Scrape PokeDex and display in tkinter

Question

Hi I am new here and I just completed my first working version of a pokedex app with a GUI using tkinter. I used selenium to scrape the data from pokemondb.net, and then used pandas to clean up the dataset then finished it up with a tkinter GUI. The project is still in early stages and can be improved in the front and backend. However, I do not have a specific question. I simply would like to receive feedback and an honest opinion of the project so far in the development process. I would be happy for any suggestions on improving the user interface also, and making it more aesthetically appealing as well. I have the entire project posted on github, however I am not sure if posting the link would be allowed here, so instead I'll post the code here.

#pokedex.py
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import sys
import os
import numpy as np
import pandas as pd
import matplotlib as plt
file_path = 'drivers/chromedriver.exe'
driver = webdriver.Chrome(file_path) # assign the driver path to variable
driver.execute("get", {'url': 'https://pokemondb.net/pokedex/all'})
driver.minimize_window() # minimize window
data = WebDriverWait(driver, 20).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "table#pokedex"))).get_attribute("outerHTML")
df = pd.read_html(data)
df = df[0]
driver.close() # close driver, end session
columns = ['id', 'name', 'type', 'total', 'hp', 'attack', 'defense', 'special-attack', 'special-defense', 'speed'] # column names (labels) for dataset
df.set_index("#", inplace=True) # set the id column to be the index
os.makedirs('C:/Users/Salah/Documents/apps/pokedex-app/datasets', exist_ok=True) # create new folder in project directory to store dataset 
df.to_csv("datasets/pokedex.csv", index=False) # converts dataset to csv file
df = pd.read_csv('datasets/pokedex.csv') # read and save file and check the first 5 entries
df.head()
for name in df.Name:
 if name.__contains__('Mega '):
 mega_id = df.index[df['Name'] == name].tolist()
 #print(mega_id)
 df.Name[mega_id] = name.split(' ', 1)[1]
poke_list = list(df.Name[:])
poke_list
df.head(10)

#gui.py
from os import times
import tkinter as tk
#from tkinter import ttk
from PIL import ImageTk, Image
from turtle import color, width
from xmlrpc.client import loads
from pokedex import *
#window
window = tk.Tk() #main
window.title('Universal Pokedex')
window.configure(background="red")
window.iconbitmap("images/icon.ico")
bg_image = tk.PhotoImage(file="images/pokedex.gif")
canvas = tk.Canvas(window, height=700, width=800)
canvas.pack()
frame = tk.Frame(window, bg='red')
frame.place(relwidth=1, relheight=1)
#window
def validate_text():
 global entry
 string = entry.get()
 #label_2.configure(text=string)
 string = string.title() # capitalize first letter of entry to match database
 if string not in poke_list: # ensure the user is typing in valid input, if not create a validation loop by reiterating user to give valid pokemon name or id.
 label_2.configure(text="That pokemon was not found, please try again: ") # validation look/error trap
 string = string.title()
 if string in poke_list:
 print("Searching for {} ... ".format(string))
 print("{} Found!".format(string))
 poke_stats = df.loc[df['Name'] == string] # return pokemon stats to user if successfully located
 label_2.configure(text=poke_stats)
#phsyical properties
label_1 = tk.Label(window, text="Welcome to the Universal Pokedex! This program contains the statistics of all the pokemon stored in the official Pokemon Database. \n Begin by entering a pokemon name below.", bg='red', font=("Cambria", 25))
label_1.place(relx=0, rely=0.05, relwidth=1, relheight=0.1)
label_2 = tk.Label(window, text="Enter a pokemon name to see a list of its stats: ", bg='#5CB3FF', font=("Courier", 20))
label_2.place(relx=0, rely=0.78, relwidth=1, relheight=0.1)
img1 = ImageTk.PhotoImage(Image.open("images/johto-starters.gif"))
img2 = ImageTk.PhotoImage(Image.open("images/pokeball.gif"))
label_3 = tk.Label(window, bg='red', image=img1) # This label will contain the entered pokemon's image
label_3.place(relx=0.25, rely=0.25, relwidth=0.50, relheight=0.50)
label_4 = tk.Label(window, bg='white', image=img2)
label_4.place(relx=0.05, rely=0.3, relwidth=0.2, relheight=0.3)
label_5 = tk.Label(window, bg='white', image=img2)
label_5.place(relx=0.75, rely=0.3, relwidth=0.2, relheight=0.3)
entry = tk.Entry(window, bg='white')
entry.place(relx=0.42, rely=0.89, relwidth=0.15, relheight=0.04)
button = tk.Button(window, text="Search", command=validate_text, bg='grey')
button.place(relx=0.42, rely=0.94, relwidth=0.15, relheight=0.05)
#physical properties
#widgets
#widgets
window.mainloop() #mainloop
```

• Reinderien: Have you read codereview.stackexchange.com/q/263671/25834 ?
• Reinderien: With a grain of salt, because I believe your use of Pandas read_html to be better than the approach in that question.
• Salah Zahran: No I haven't read that until now. And do you think the code could use some work? Also that was a little advice I received on stack overflow, but I do see how he could have used it better in that example.
• Reinderien: Yes, it could use work; I'm writing up an answer.

Answer 1

Selenium is not necessary for this project so don't use it. You can go direct to requests. This website doesn't have an API but others do; you should prefer them instead. Even better, you should have an offline database embedded in your program since it never changes. Perhaps that's why you read_csv; this would be more useful if you first check for the existence of a CSV and only scrape in the application if it doesn't exist.

These:

import sys
import os
import numpy as np
import matplotlib as plt

aren't needed, so delete them. A decent IDE will show you which imports are unneeded.

Move your global code into functions and/or classes.

There isn't much point in redefining your column names - the defaults as scraped from Pandas are actually nicer for presentation purposes.

Don't use for loops in Pandas. Your string operation is simple and vectorisable. Don't cast to a list.

Your colours make my eyes bleed. Call me boring, but it's rarely justifiable to override colours and fonts.

Your layout can be simplified by use of grid calls.

You don't need a button if you search whenever the user updates their search term in the entry box.

You should make better use of tk StringVar to decouple your data from your UI.

Future improvements could include calling into a fuzzy-matching library to do inexact string search, and displaying your search results in multiple rows in a real tkinter TreeView instead of dumping them to a string.

Suggested

Covering some of the above,

import tkinter as tk
from threading import Thread
from typing import Optional
import pandas as pd
import requests
def load_data() -> pd.DataFrame:
 print('Retrieving data...')
 with requests.get(
 'https://pokemondb.net/pokedex/all',
 headers={'User-Agent': 'Mozilla/5.0'},
 ) as response:
 response.raise_for_status()
 df, = pd.read_html(
 io=response.text, flavor='bs4', index_col='#',
 )
 df['SearchKey'] = df.Name.str.lower()
 df[['Name', 'Mega']] = df.Name.str.split(' Mega ', n=1, expand=True)
 return df
class GUI:
 def __init__(self) -> None:
 window = tk.Tk()
 window.title('Universal Pokedex')
 self.run = window.mainloop
 frame = tk.Frame(window)
 frame.pack()
 tk.Label(
 frame,
 text='Welcome to the Universal Pokedex! This program contains the '
 'statistics of all the pokemon stored in the official Pokemon '
 'Database.\n'
 'Begin by entering a pokemon name below.',
 justify='left', wraplength=200,
 ).grid(row=0, column=0, columnspan=2)
 tk.Label(
 frame, text='Pokemon name: ',
 ).grid(row=1, column=0)
 self.search_term = tk.StringVar(frame, name='search_term')
 self.entry = tk.Entry(
 frame, textvariable=self.search_term, state='disabled',
 )
 self.entry.grid(row=1, column=1)
 self.result = tk.StringVar(frame, name='result', value='Loading...')
 tk.Label(
 frame, textvariable=self.result,
 ).grid(row=2, column=0, columnspan=2)
 self.data: Optional[pd.DataFrame] = None
 def set_data(self, data: pd.DataFrame) -> None:
 self.data = data
 self.result.set('')
 self.entry.configure(state='normal')
 self.search_term.trace_add('write', self.search)
 def search(self, name: str, _, mode: str) -> None:
 term = self.search_term.get().lower()
 if term:
 predicate = self.data.SearchKey.str.contains(term)
 matches = self.data[predicate].iloc[:1]
 if len(matches):
 first = matches.iloc[0, :]
 self.result.set(str(first))
 return
 self.result.set('')
def main() -> None:
 gui = GUI()
 def fetch_data() -> None:
 data = load_data()
 gui.set_data(data)
 data_thread = Thread(target=fetch_data)
 data_thread.start()
 gui.run()
if __name__ == '__main__':
 main()

screencapture, squirtle

• Salah Zahran: Thank you so much! This is exactly what I was looking for. I was sure there would be many flaws that could be detected from an objective eye, so I thank you for the constructive criticism, and will work toward improvements
• Salah Zahran: by the way, just curious about the offline database you mentioned. Are you refer to using a sql database? Would I insert the pokemon data into the db by using an api on another website that has one like you said?
• Reinderien: "Offline database" can be as simple as a static CSV file shipped with your application. The application probably doesn't call for more than that.