from threading import Thread
from Queue import Queue
import random, time
from bs4 import BeautifulSoup
import os, sys, urllib2
import urllib2,os,socket
import thread


'''
 date: 2014年03月07日 20:50:15
 url: http://www.dbmeizi.com
 desc: using mulitithreading download pictures from dbmeizi.com
 email: withfaker@gmail.com
'''

queue = Queue()
pic_path = os.path.join(os.curdir, "images")

class ProducerThread(Thread):
 def run(self):
 page_loop()

class ConsumerThread(Thread):
 def run(self):
 while True:
 if queue.empty():
 thread.exit()
 url = queue.get()
 queue.task_done()
 fetch(url) 

def page_loop(page=0):
 url = 'http://www.dbmeizi.com/?p=%s' % page
 try:
 content = urllib2.urlopen(url)
 soup = BeautifulSoup(content)
 except:
 print "internal error:[%s]" % url
 page_loop(int(page)+1)

 my_girl = soup.find_all('img')
 if my_girl == []:
 print 'finished!'
 sys.exit(0)
 #print "BEGIN TO FETCH PAGE:[%s]" % page
 for girl in my_girl:
 link = girl.get('src')
 flink = 'http:' + link
 queue.put(flink)
 page = int(page) + 1
 page_loop(page)

#fetch pictures
def fetch(url):
 print "url:[%s]" % url
 p = os.path.join(os.curdir, pic_path, url[-11:])
 if os.path.isfile(p):
 st = os.stat(p)
 if st.st_size > 0:
 print "file[%s] is already exists." % url[-11:]
 return
 else:
 print "file[%s] exists. but size is too small." % url[-11:]
 req = urllib2.Request(url)
 req.add_header('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8')
 req.add_header('Accept-Encoding','gzip,deflate,sdch')
 req.add_header('Accept-Language','zh-CN,zh;q=0.8,en;q=0.6')
 req.add_header('Cache-Control','max-age=0')
 req.add_header('Connection','keep-alive')
 req.add_header('Referer','https://www.dbmeizi.com')
 req.add_header('User-Agent','Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1660.0 Safari/537.36')
 try:
 resp = urllib2.urlopen(req)
 except:
 print "internal error:[%s]" % url
 return
 f = open(p, 'wb')
 f.write(resp.read())
 f.close()
 resp.close()
 #print "fetch url done.[%s]" % url

if __name__ == '__main__':
 if not os.path.isdir(pic_path):
 os.mkdir(pic_path)
 ProducerThread(name="Producer1").start()
 ConsumerThread(name="Consumer1").start()
 ConsumerThread(name="Consumer2").start()
 ConsumerThread(name="Consumer3").start()
 ConsumerThread(name="Consumer4").start()
 ConsumerThread(name="Consumer5").start()
 #ConsumerThread(name="Consumer6").start()
 #ConsumerThread(name="Consumer7").start()
 #ConsumerThread(name="Consumer8").start()
 #ConsumerThread(name="Consumer9").start()
 #ConsumerThread(name="Consumer10").start()