• 1

• 243
• 3
• 13

This is now my new code implementing the recommendations from my previous question.

import os
import time
import mysql.connector
import MySQLdb
import ConfigParser
import base64
import logging
from logging.handlers import TimedRotatingFileHandler
import sys
import ast
import smtplib
from sendmail import send_mail
"""
TO DO:
1. Retry failure upload when mysql disconnects
"""
currdir = os.getcwd()
curr_datetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
"""function that reads the config file"""
def read_Config(section, option):
 conf_file = currdir + "\\Config\\config.ini"
 config = ConfigParser.ConfigParser()
 config.read(conf_file)
 sections = config.sections()
 
 def ConfigSectionMap(section):
 dict1 = {}
 options = config.options(section)
 for option in options:
 try:
 dict1[option] = config.get(section, option)
 if dict1[option] == -1:
 # print("skip: %s" % option)
 LOGGER.debug("skip: %s" % option)
 except:
 deb = "exception on %s!" % option
 LOGGER.debug(deb)
 write_log(wyko_no,'DEBUG',deb,cur_datetime)
 dict1[option] = None
 return dict1
 
 res = ConfigSectionMap(section)[option]
 return res
"""read configuration file"""
app_name = read_Config('application','app_name')
FILE_PATH = read_Config('application','file_path')
LOG_PATH = currdir + "\\" + read_Config('application','log_path') + "\\"
USER = read_Config('database','user')
PASSWORD = read_Config('database','password')
PASSWORD = base64.decodestring(PASSWORD) ##decode password, remove D and L from the string
HOST = read_Config('database','host')
DB = read_Config('database','db')
DB_PORT = read_Config('database','port')
RECEIVERS = ast.literal_eval(read_Config('email','receivers'))
SENDER = read_Config('email','sender')
SMTP = read_Config('email','smtp')
PORT = read_Config('email','port')
"""set up logger"""
LOGGER = logging.getLogger(app_name)
LOG_FILE = app_name + '.log'
LOG_PATH = LOG_PATH + LOG_FILE
LOGGER.setLevel(logging.DEBUG)
hdlr = TimedRotatingFileHandler(LOG_PATH, when="d", interval=1, backupCount=1)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s: %(message)s')
hdlr.setFormatter(formatter)
LOGGER.addHandler(hdlr) 
sh = logging.StreamHandler(sys.stdout)
sh.setLevel(logging.DEBUG)
sh.setFormatter(formatter)
LOGGER.addHandler(sh)
 
"""insert csv to database"""
def insert_csv(file,filename):
 try:
 cnx = mysql.connector.connect(user = USER,
 password = PASSWORD,host = HOST,
 database = DB, port = DB_PORT)
 cursor = cnx.cursor() 
 with open(file, 'rb') as f:
 thedata = f.read()
 curr_datetime = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
 sql = "INSERT INTO wyko_file(wyko_file,file_name,date_inserted) VALUES (%s,%s,%s)"
 cursor.execute(sql,(thedata,filename,curr_datetime))
 if cursor.rowcount != 0:
 LOGGER.info("Transfer to database succesful!")
 cnx.commit() 
 except (MySQLdb.OperationalError, MySQLdb.ProgrammingError), e:
 LOGGER.error(e)
 send_mail(SENDER,RECEIVERS,e)
 finally:
 cursor.close()
 cnx.close()
 
 
"""main loop that watches the directory""" 
def main():
 try:
 for dirpath, dirnames, files in os.walk(FILE_PATH):
 for i in files:
 file = dirpath+i
 try:
 LOGGER.info("Transferring file: " +i)
 insert_csv(file,i)
 except Exception, e:
 LOGGER.error(e)
 send_mail(SENDER,RECEIVERS,e)
 finally:
 os.remove(file)
 LOGGER.info('File successfully removed from '+FILE_PATH+"\n"+"-"*80)
 except Exception, e:
 LOGGER.error(e)
 send_mail(SENDER,RECEIVERS,e)
 # break
 
if __name__ == "__main__":
 LOGGER.info("Application starts running\n"+"-"*80)
 main()

But upon deploying this application on a client, I have encountered an issue that this parser is eating up the resources on the client computer and causes applications to hang/lag. I think the code needs optimizing.

The parser is currently deployed on a Windows XP OS with 2GB of RAM.