# -*- coding: utf-8 -*-
"""
Created on Fri Mar 29 14:24:32 2013
@author: waventropy
"""
import urllib2
import bs4 as bs
import sqlite3
import re
#定义补足为给出的网址头部分
aa='http://patft.uspto.gov/'
g=open('new11.txt','wr')
#创建数据库,存储数据
conn = sqlite3.connect("Patentdatabase.db")
cursor = conn.cursor()
cursor.execute("""CREATE TABLE Patent
 (PatentNumber text,PatentName text,PatentInventors text,PatentCompany text,PatentFiledtime text,PatentAbstract text) """)
#给出其实搜索地址。主要是x-ray和detect
#然后将参数传给soup

starturl = 'http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&Sect2=HITOFF&p=1&u=%2Fnetahtml%2FPTO%2Fsearch-bool.html&r=0&f=S&l=50&TERM1=X+ray&FIELD1=ABTX&co1=AND&TERM2=detect&FIELD2=ABTX&d=PTXT'
#添加浏览器信息防止服务器中止频繁项请求

request = urllib2.Request(starturl)
request.add_header('User-agent', 'Mozilla/5.0 (Linux i686)')
response = urllib2.urlopen(request).read()
soup = bs.BeautifulSoup(response)
#得到所有的检索结果
number=soup.findAll('i')
AllNumber=number[1].findAll('strong')[2].text
AllNumber=int(AllNumber)
#找到所有的记录

#设立计数变量因为USPTO网页的显示限制是每次50条记录
n=abs(AllNumber-(AllNumber/50)*50)
if n==0:
 CircleNumber=(AllNumber/50)
else:
 CircleNumber=(AllNumber/50)+1
for k in range(1,CircleNumber):
 tr=soup.findAll('tr')
 for i in range(2,len(tr)-2):
 temp=tr[i].findAll('td')
 #在次页面提取PatentName信息
 PatentName=temp[3].text.strip()
 #重新进入新的搜索出的结果页面进行数据提取
 urltemp=aa+temp[1].a.get('href')
 request = urllib2.Request(urltemp)
 request.add_header('User-agent', 'Mozilla/5.0 (Linux i686)')
 response = urllib2.urlopen(request).read()
 souptemp = bs.BeautifulSoup(response)
 PatentNumber=souptemp.html.head.title.renderContents().strip()
 #PatentInventors使用正则化搜索避开可能的不规则排列顺序
 anchortemp = souptemp.findAll(text=re.compile("Inv"))
 if len(anchortemp)>0:
 anchortemp = anchortemp[0]
 PatentInventors=' '.join(anchortemp.find_next('td').stripped_strings)
 else:
 PatentInventors='NoRecord'
 #PatentCompany使用正则化搜索避开可能的不规则排列顺序
 anchortemp = souptemp.findAll(text=re.compile("Assi"))
 if len(anchortemp)>0:
 anchortemp = anchortemp[0]
 PatentCompany=' '.join(anchortemp.find_next('td').stripped_strings)
 else:
 PatentCompany='NoRecord'
 #PatentFiledtime使用正则化搜索避开可能的不规则排列顺序
 anchortemp = souptemp.findAll(text=re.compile("File"))
 if len(anchortemp)>0:
 anchortemp = anchortemp[0]
 PatentFiledtime= ' '.join(anchortemp.find_next('td').stripped_strings)
 
 else:
 PatentFiledtime='NoRecord'
 #对于PatentAbstract进行格式化
 PatentAbstract=souptemp.html.body.p.text.strip()
 PatentAbstract.replace('\n', '')
 PatentAbstract=' '.join(PatentAbstract.split())
 #向数据库中写入记录
 cursor.execute("INSERT INTO Patent VALUES (?,?,?,?,?,?)" ,(PatentNumber,PatentName,PatentInventors,PatentCompany,PatentFiledtime,PatentAbstract))
 conn.commit()
 g.write("%s \n"%(PatentInventors))
 ktemptemp=str(k)
 urltemptemp='http://patft.uspto.gov/netacgi/nph-Parser?Sect1=PTO2&Sect2=HITOFF&u=%2Fnetahtml%2FPTO%2Fsearch-adv.htm&r=0&f=S&l=50&d=PTXT&OS=ABST%2F%22X+ray%22+AND+ABST%2Fdetect&RS=ABST%2F%22X+ray%22+AND+ABST%2Fdetect&Query=ABST%2F%22X+ray%22+AND+ABST%2Fdetect&TD=352&Srch1=%22X+ray%22.ABTX.&Srch2=detect.ABTX.&Conj1=AND&NextList'+ktemptemp+'=Next+50+Hits'
 #添加浏览器信息防止服务器中止频繁项请求
 request1 = urllib2.Request(urltemptemp)
 request1.add_header('User-agent', 'Mozilla/5.0 (Linux i686)')
 response1 = urllib2.urlopen(request1).read()
 soup = bs.BeautifulSoup(response1)
g.close()
print 'Done'