Problem in Handling MySql Data.

subhabrata.banerji at gmail.com subhabrata.banerji at gmail.com
Sat May 2 06:42:14 EDT 2015


Dear Group,
I am trying to write the following script to call around 0.3 million files from a remote server. 
It is generally working fine, but could work only upto 65 to 70 files. After this, it is just
printing the file names and not processing anything. If anyone may kindly suggest what I am
doing wrong?
import pymysql
import pymysql.cursors
import os
import win32com.client
from gensim.models import Word2Vec
import nltk
from nltk.corpus import stopwords
import pyPdf
from pyth.plugins.rtf15.reader import Rtf15Reader
from pyth.plugins.plaintext.writer import PlaintextWriter
import nltk
import zipfile, re
import time
#READING ONE DOC FILE FROM REMOTE LOCATION
def readfilesq9(n):
 connection = pymysql.connect(host='xxx.xxx.x.xxx',
 user='abcd',
 passwd='pwd1',
 db='rep_db',
 charset='utf8mb4',
 cursorclass=pymysql.cursors.DictCursor)
 list1=[]
 with connection.cursor() as cursor:
 # Read a single record
 sql = "SELECT candidateid,cnd.FirstName, cnd.LastName,Concat('\\xxx.xxx.x.xxx\File\Cand_Res/',orgguid,'/',DATE_FORMAT(cnd.createddate,'%Y%m'),'/',candidateguid,'/',Resume) as ResumePath from candidate cnd join mstorganization org on cnd.orgid = org.OrgId where Resume <> '' and Resume is not null order by cnd.modifieddate limit 100000"
 cursor.execute(sql)
 result = cursor.fetchall()
 #print result
 #list1=[]
 for i in result:
 try:
 #print i
 item_1=i.items()
 item_2=item_1[2][1]
 print item_2
 item_3=item_2.index("/")
 file1=item_2[item_2:]
 string1='\\\\xxx.xxx.x.xxx\\Resumes\\Cand_Res'
 file1e=file1.encode('ascii', 'ignore')
 urls=file1e.replace("/","\\")
 file_full=string1+urls
 time.sleep(1)
 #osp="C:\\Python27"
 os1=os.path.abspath(os.curdir)
 osp2=os.path.join(os1,file_full)
 print "Path1:",osp2
 file_name1=osp2
 print "Path:",file_name1
 #IDENTIFICATION OF FILE KIND
 #DOC CONVERSION
 if ".doc" in file_name1:
 #EXTRACTING ONLY .DOC FILES
 if ".docx" not in file_name1:
 #print "It is A Doc file$$:",file_name
 try:
 doc = win32com.client.GetObject(file_name1)
 text = doc.Range().Text
 text1=text.encode('ascii','ignore')
 text_word=text1.split()
 #print "The Text Word is:",text_word
 #print "Text for Document File Is:",text1
 list1.append(text_word)
 #print "List for Doc File Is:",list3
 #print "It is a Doc file"
 except:
 print "DOC ISSUE"
 #EXTRACTING ONLY .DOCX FILES
 elif ".docx" in file_name1:
 #print "It is DOCX FILE:",file_name
 docx1=zipfile.ZipFile(file_name1)
 content = docx1.read('word/document.xml').decode('utf-8')
 cleaned = re.sub('<(.|\n)*?>','',content).encode('ascii','ignore')
 cleaned_word=cleaned.split()
 #print "The Cleaned Document Is:",cleaned
 list1.append(cleaned_word)
 #print "List for DocX file Is:",list4
 else:
 print "NONE1"
 else:
 print "It is not a Doc file" 
 
 
 except:
 print "OOPS1"
I am using Python2.7.6 on Enthought Canopy. It is not my default Python. My default Python is in location,"C:\\Python27".
I am using MySql and Windows 7 Professional. Apology for any indentation error.
Regards,
Subhabrata Banerjee. 


More information about the Python-list mailing list

AltStyle によって変換されたページ (->オリジナル) /