Make a loop faster when reading large text files

Question

im developing a program that will read text files and convert it to JSON format dynamically. Users are able to input the key name and their respective digit to slice the index from a raw text file. Everything is working fine when reading a small size file, but whenever the text files is huge, the program will load like a few minute in order to proceed. I'm new to programming and have no experience to make a code more "efficient". What my goal is to make the loop process (read file) to be more faster. Does anyone could help me ? Thanks in advance.

My huge text files is around 200kb and 2000+ lines of log

My program: enter image description here

After user click on "Review JSON" , the program will convert the text file to json format based on the index slices and print into terminal (this is where the process that I want to make it more faster) enter image description here

My Code:

#Review JSON format
def ReviewJson():
 #if no selection is choosed
 if x == 183.0:
 tkMessageBox.showinfo("Error", "You must select a file before can review.", icon="error")
 else:
 # ========valdiate buttons for create new profile
 try:
 ReviewJSON_button.configure(state=DISABLED)
 # Delete submit button when select existing value
 CreateJsonBtn.destroy()
 CreateNewJsonBtn.configure(state=NORMAL)
 except:
 pass
 global reviewjson, window
 window = Toplevel(root)
 window.minsize(800, 600)
 window.maxsize(800, 600)
 window.title("Foghorn Publisher - Review Json" + " " + "(" + options.get() + ")")
 window.protocol('WM_DELETE_WINDOW', destroyReviewJSON)
 reviewjson = Text(window, bg='black', foreground="white", height="20", width="500")
 reviewjson.pack()
 reviewjson.config(state=NORMAL)
 reviewjson.delete("1.0", "end")
 file_data.clear()
 try:
 global datalist,cleandatalist
 datalist=[]
 for content in data2:
 #Key1name cant be empty
 if (key1EntryName.get()==""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error","First key name cannot be empty",icon="error")
 #start index and end index cannot be empty after name is been declared
 elif(key1EntryName.get() != 0) and (key1EntryStartIndex.get() =="") and (key1EntryEndIndex.get() ==""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Start index and end index cannot be empty after declaration of key name", icon="error")
 # check 1: check start to EOS
 elif (key1EntryEndIndex.get()) == "":
 file_data[key1EntryName.get()] = content[int(key1EntryStartIndex.get()):]
 # check 1: check EOS to start
 elif (key1EntryStartIndex.get()) == "":
 file_data[key1EntryName.get()] = content[:int(key1EntryEndIndex.get())]
 # check 1: normal status
 else:
 file_data[key1EntryName.get()] = content[int(key1EntryStartIndex.get()):int(key1EntryEndIndex.get())]
 ######################Check 2 ################################
 #check 2: If all empty jiu dont call this part
 if(key2EntryName.get() or key2EntryStartIndex.get() or key2EntryEndIndex.get()) == "":
 pass
 #check 2: start index and end index cannot be empty after name is been declared
 elif(key2EntryName.get() != 0) and (key2EntryStartIndex.get() =="") and (key2EntryEndIndex.get() ==""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Start index and end index cannot be empty after declaration of key name", icon="error")
 elif (key2EntryName.get() == "") and (key2EntryStartIndex.get() != 0) and (
 key2EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of start index",
 icon="error")
 elif (key2EntryName.get() == "") and (key2EntryStartIndex.get() == "") and (
 key2EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of end index",
 icon="error")
 elif (key2EntryName.get() == "") and (key2EntryStartIndex.get() != 0) and (
 key2EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of start & end index", icon="error")
 # check 2: check start to EOS
 elif (key2EntryEndIndex.get()) == "":
 file_data[key2EntryName.get()] = content[int(key2EntryStartIndex.get()):]
 # check 2: check EOS to start
 elif (key2EntryStartIndex.get()) == "":
 file_data[key2EntryName.get()] = content[:int(key2EntryEndIndex.get())]
 # check 2: normal status
 else:
 file_data[key2EntryName.get()] = content[int(key2EntryStartIndex.get()):int(key2EntryEndIndex.get())]
 ######################Check 3 ################################
 # check 3: If all empty jiu dont call this part
 if (key3EntryName.get() or key3EntryStartIndex.get() or key3EntryEndIndex.get()) == "":
 pass
 # check 3: start index and end index cannot be empty after name is been declared
 elif (key3EntryName.get() != 0) and (key3EntryStartIndex.get() == "") and (
 key3EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Start index and end index cannot be empty after declaration of key name",
 icon="error")
 elif (key3EntryName.get() == "") and (key3EntryStartIndex.get() != 0) and (
 key3EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of start index",
 icon="error")
 elif (key3EntryName.get() == "") and (key3EntryStartIndex.get() == "") and (
 key3EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of end index",
 icon="error")
 elif (key3EntryName.get() == "") and (key3EntryStartIndex.get() != 0) and (
 key3EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Key name cannot be empty after declaration of start & end index",
 icon="error")
 # check 3: check start to EOS
 elif (key3EntryEndIndex.get()) == "":
 file_data[key3EntryName.get()] = content[int(key3EntryStartIndex.get()):]
 # check 3: check EOS to start
 elif (key3EntryStartIndex.get()) == "":
 file_data[key3EntryName.get()] = content[:int(key3EntryEndIndex.get())]
 # check 3: normal status
 else:
 file_data[key3EntryName.get()] = content[
 int(key3EntryStartIndex.get()):int(key3EntryEndIndex.get())]
 ######################Check 4 ################################
 # check 4: If all empty jiu dont call this part
 if (key4EntryName.get() or key4EntryStartIndex.get() or key4EntryEndIndex.get()) == "":
 pass
 # check 4: start index and end index cannot be empty after name is been declared
 elif (key4EntryName.get() != 0) and (key4EntryStartIndex.get() == "") and (
 key4EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Start index and end index cannot be empty after declaration of key name",
 icon="error")
 elif (key4EntryName.get() == "") and (key4EntryStartIndex.get() != 0) and (
 key4EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of start index",
 icon="error")
 elif (key4EntryName.get() == "") and (key4EntryStartIndex.get() == "") and (
 key2EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error", "Key name cannot be empty after declaration of end index",
 icon="error")
 elif (key4EntryName.get() == "") and (key4EntryStartIndex.get() != 0) and (
 key4EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Key name cannot be empty after declaration of start & end index",
 icon="error")
 # check 4: check start to EOS
 elif (key4EntryEndIndex.get()) == "":
 file_data[key4EntryName.get()] = content[int(key4EntryStartIndex.get()):]
 # check 4: check EOS to start
 elif (key4EntryStartIndex.get()) == "":
 file_data[key4EntryName.get()] = content[:int(key4EntryEndIndex.get())]
 # check 4: normal status
 else:
 file_data[key4EntryName.get()] = content[int(key4EntryStartIndex.get()):int(
 key4EntryEndIndex.get())]
 ######################Check 5 ################################
 # check 5: If all empty jiu dont call this part
 if (key5EntryName.get() or key5EntryStartIndex.get() or key5EntryEndIndex.get()) == "":
 pass
 # check 5: start index and end index cannot be empty after name is been declared
 elif (key5EntryName.get() != 0) and (key5EntryStartIndex.get() == "") and (
 key5EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Start index and end index cannot be empty after declaration of key name",
 icon="error")
 elif (key5EntryName.get() == "") and (key5EntryStartIndex.get() != 0) and (
 key5EntryEndIndex.get() == ""):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Key name cannot be empty after declaration of start index",
 icon="error")
 elif (key5EntryName.get() == "") and (key5EntryStartIndex.get() == "") and (
 key5EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Key name cannot be empty after declaration of end index",
 icon="error")
 elif (key5EntryName.get() == "") and (key5EntryStartIndex.get() != 0) and (
 key5EntryEndIndex.get() != 0):
 window.destroy()
 ReviewJSON_button.configure(state=NORMAL)
 tkMessageBox.showinfo("Error",
 "Key name cannot be empty after declaration of start & end index",
 icon="error")
 # check 5: check start to EOS
 elif (key5EntryEndIndex.get()) == "":
 file_data[key5EntryName.get()] = content[int(key5EntryStartIndex.get()):]
 # check 5: check EOS to start
 elif (key5EntryStartIndex.get()) == "":
 file_data[key5EntryName.get()] = content[:int(key5EntryEndIndex.get())]
 # check 5: normal status
 else:
 file_data[key5EntryName.get()] = content[int(key5EntryStartIndex.get()):int(
 key5EntryEndIndex.get())]
 # output to JSON
 global tmp
 tmp = json.dumps(file_data, ensure_ascii=False, indent="\t")
 datalist.append(tmp)
 # We want to strip all elements in this list
 clearslashN = [i.replace('\n','') for i in datalist]
 cleandatalist = [i.replace('\t', '') for i in clearslashN]
 print(cleandatalist)
 except:
 raise
```

Answer 1

• Keys names, and start and end indices do not ever change. It makes no sense to test them at each iteration. Move all the error checking fragments outside of the loop.

  Same applies to content slicing. Compute the boundaries beforehand.

• The first line of your data file is aligned differently. I bet its json is incorrect.

  Instead of slicing content at columns, use a regex, e.g.

   r'([^ ]+) ([^;]+);([^ ]+) ([^ ]+) (.*)$'
  

• Do not strip the entire datalist over and over again. Only strip tmp, before adding it to the datalist.

• Jackson Lim: the reason why I need to have user input for key names and start,end index is because user are able to choose different raw files to be convert and not only this files.