wind_crawl.py · df270464/python-learn

代码拉取完成,页面将自动刷新

扫描微信二维码支付

取消

支付完成

richgiteeai

Watch

不关注关注所有动态仅关注版本发行动态关注但不提醒动态

1 Star 0 Fork 265

df270464/python-learn

forked from mktime/python-learn

代码 Issues 0 Pull Requests 0 Wiki 统计流水线

服务

加入 Gitee

与超过 1400万开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)

免费加入

已有帐号? 立即登录

文件

master

分支 (1)

管理

master

克隆/下载

HTTPS SSH SVN SVN+SSH 下载ZIP

提示

下载代码请复制以下命令到终端执行

为确保你提交的代码身份被 Gitee 正确识别,请执行以下命令完成配置

git config --global user.name userName 
git config --global user.email userEmail

初次使用 SSH 协议进行代码克隆、推送等操作时,需按下述提示完成 SSH 配置

1 生成 RSA 密钥

2 获取 RSA 公钥内容,并配置到 SSH公钥中

在 Gitee 上使用 SVN,请访问使用指南

使用 HTTPS 协议时,命令行会出现如下账号密码验证步骤。基于安全考虑,Gitee 建议配置并使用私人令牌替代登录密码进行克隆、推送等操作

Username for 'https://gitee.com': userName

Password for 'https://userName@gitee.com': # 私人令牌

分支 1

标签 0

python-learn

wind_crawl.py

wind_crawl.py 29.44 KB

# -*- encoding=utf-8 -*-
from WindPy import *
import datetime
import os
import copy
import paramiko
import base64
import configparser
import time

config = configparser.ConfigParser()
config.read('wind.ini')
HOST = config['FTP']['host']
PORT = int(config['FTP']['port'])
USER = config['FTP']['user']
PASSWORD = config['FTP']['password']
TIMEOUT = int(config['FTP']['timeout'])
LOCAL_PATH = config['FTP']['local_path']
SERVER_PATH = config['FTP']['server_path']
WIND_USERNAME = config['WIND']['username']
WIND_PASSWORD = config['WIND']['password']
log_file = config['LOG']['log_file']

class FtpUtil:
 def __init__(self, host, port, user, passwd, timeout=10):
 self.server_path = ''
 self.local_path = ''
 _ = paramiko.Transport((host, port))
 _.banner_timeout = timeout
 _.connect(username=user, password=passwd)
 self.sftp = paramiko.SFTPClient.from_transport(_)

def set_server_path(self, p):
 self.server_path = p

def set_local_path(self, p):
 self.local_path = p
 
 def download(self, fname):
 server_path = self.server_path + '/' + fname
 local_path = os.path.join(self.local_path,fname)
 self.sftp.get(server_path, local_path)

def downloads(self, l):
 for item in l:
 self.download(item)

def upload(self, fname):
 server_path = self.server_path + '/' + fname
 local_path = os.path.join(self.local_path,fname)
 self.sftp.put(local_path, server_path)

def log(msg):
 timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
 thread_name = threading.current_thread().getName()
 f = open(log_file, 'a')
 f.write('[' + timestamp + '][' + thread_name + ']' + msg + '\n')
 f.close()

fields_config = {
 "stock": {
 "type": "stock",
 "fields":"windcode,sec_name,volume,amt,turn,open,high,low,pre_close",
 "output": "WDPYTHON_STOCK_YYYYMMDD.txt",
 "len": 9
 },
 "fund":{
 "type": "fund",
 "fields": "windcode,fund_fullname,name_official,fund_type,nav,fund_navcur,nav_acc,fund_benchmark",
 "output": "WDPYTHON_FUND_YYYYMMDD.txt",
 "len": 8
 },
 "bond":{
 "type": "bond",
 "fields": "windcode,fullname,sec_name,exch_city,windl1type,windl2type,chinabondl1type,chinabondl2type,shclearl1type,yield_cnbd,net_cnbd,dirty_cnbd,yield_csi1,net_csi1,dirty_csi1",
 "output": "WDPYTHON_BOND_YYYYMMDD.txt",
 "len": 15
 },
 "rating": {
 "type": "rating",
 "fields": "windcode,issuerupdated,comp_name,registernumber,organizationcode,trade_code,amount,rate_latest,rate_changesofrating,rate_style,latestissurercreditrating,latestissurercreditratingdate,latestissurercreditratingtype",
 "output": "WDPYTHON_BOND_RATING_YYYYMMDD.txt",
 "len":13
 },
 "house": {
 "type": "house",
 "codes": ['S0147356','S0147357','S0147358','S0147359','S0147360','S0147361','S0147362','S0147363','S0147364','S0147365','S0147366','S0147367','S0147368','S0147369','S0147370','S0147371','S0147372','S0147373','S0147374','S0147375','S0147376','S0147377','S0147378','S0147379','S0147380','S0147381','S0147382','S0147383','S0147384','S0147385','S0147386','S0147387','S0147388','S0147389','S0147390','S0147391','S0147392','S0147393','S0147394','S0147395','S0147396','S0147397','S0147398','S0147399','S0147400','S0147401','S0147402','S0147403','S0147404','S0147405','S0147406','S0147407','S0147408','S0147409','S0147410','S0147411','S0147412','S0147413','S0147414','S0147415','S0147416','S0147417','S0147418','S0147419','S0147420','S0147421','S0147422','S0147423','S0147424','S0147425','S0147426','S0147427','S0147428','S0147429','S0147430','S0147431','S0147432','S0147433','S0147434','S0147435','S0147436','S0147437','S0147438','S0147439','S0147440','S0147441','S0147442','S0147443','S0147444','S0147445','S0147446','S0147447','S0147448','S0147449','S0147450','S0147451','S0147452','S0147453','S0147454','S0147455','S0147456','S0147457','S0147458','S0147459','S0147460','S0147461','S0147462','S0147463','S0147464','S0147465','S0147466','S0147467','S0147468','S0147469','S0147470','S0147471','S0147472','S0147473','S0147474','S0147475','S0147476','S0147477','S0147478','S0147479','S0147480','S0147481','S0147482','S0147483','S0147484','S0147485','S0147486','S0147487','S0147488','S0147489','S0147490','S0147491','S0147492','S0147493','S0147494','S0147495','S0147496','S0147497','S0147498','S0147499','S0147500','S0147501','S0147502','S0147503','S0147504','S0147505','S0147506','S0147507','S0147508','S0147509','S0147510','S0147511','S0147512','S0147513','S0147514','S0147515','S0147516','S0147517','S0147518','S0147519','S0147520','S0147521','S0147522','S0147523','S0147524','S0147525','S0147526','S0147527','S0147528','S0147529','S0147530','S0147531','S0147532','S0147533','S0147534','S0147535','S0147536','S0147537','S0147538','S0147539','S0147540','S0147541','S0147542','S0147543','S0147544','S0147545','S0147546','S0147547','S0147548','S0147549','S0147550','S0147551','S0147552','S0147553','S0147554','S0147555','S0147556','S0147557','S0147558','S0147559','S0147560','S0147561','S0147562','S0147563','S0147564','S0147565','S0147566','S0147567','S0147568','S0147569','S0147570','S0147571','S0147572','S0147573','S0147574','S0147575','S0147576','S0147577','S0147578','S0147579','S0147580','S0147581','S0147582','S0147583','S0147584','S0147585','S0147586','S0147587','S0147588','S0147589','S0147590','S0147591','S0147592','S0147593','S0147594','S0147595','S0147596','S0147597','S0147598','S0147599','S0147600','S0147601','S0147602','S0147603','S0147604','S0147605','S0147606','S0147607','S0147608','S0147609','S0147610','S0147611','S0147612','S0147613','S0147614','S0147615','S0147616','S0147617','S0147618','S0147619','S0147620','S0147621','S0147622','S0147623','S0147624','S0147625','S0147626','S0147627','S0147628','S0147629','S0147630','S0147631','S0147632','S0147633','S0147634','S0147635','S0147636','S0147637','S0147638','S0147639','S0147640','S0147641','S0147642','S0147643','S0147644','S0147645','S0147646','S0147647','S0147648','S0147649','S0147650','S0147651','S0147652','S0147653','S0147654','S0147655','S0147656','S0147657','S0147658','S0147659','S0147660','S0147661','S0147662','S0147663','S0147664','S0147665','S0147666','S0147667','S0147668','S0147669','S0147670','S0147671','S0147672','S0147673','S0147674','S0147675','S0147676','S0147677','S0147678','S0147679','S0147680','S0147681','S0147682','S0147683','S0147684','S0147685','S0147686','S0147687','S0147688','S0147689','S0147690','S0147691','S0147692','S0147693','S0147694','S0147695','S0147696','S0147697','S0147698','S0147699','S0147700','S0147701','S0147702','S0147703','S0147704','S0147705','S0147706','S0147707','S0147708','S0147709','S0147710','S0147711','S0147712','S0147713','S0147714','S0147715','S0147716','S0147717','S0147718','S0147719','S0147720','S0147721','S0147722','S0147723','S0147724','S0147725','S0147726','S0147727','S0147728','S0147729','S0147730','S0147731','S0147732','S0147733','S0147734','S0147735','S0147736','S0147737','S0147738','S0147739','S0147740','S0147741','S0147742','S0147743','S0147744','S0147745','S0147746','S0147747','S0147748','S0147749','S0147750','S0147751','S0147752','S0147753','S0147754','S0147755','S0147756','S0147757','S0147758','S0147759','S0147760','S0147761','S0147762','S0147763','S0147764','S0147765','S0147766','S0147767','S0147768','S0147769','S0147770','S0147771','S0147772','S0147773','S0147774','S0147775','S0147776','S0147777','S0147778','S0147779','S0147780','S0147781','S0147782','S0147783','S0147784','S0147785','S0147786','S0147787','S0147788','S0147789','S0147790','S0147791','S0147792','S0147793','S0147794','S0147795','S0147796','S0147797','S0147798','S0147799','S0147800','S0147801','S0147802','S0147803','S0147804','S0147805','S0147806','S0147807','S0147808','S0147809','S0147810','S0147811','S0147812','S0147813','S0147814','S0147815','S0147816','S0147817','S0147818','S0147819','S0147820','S0147821','S0147822','S0147823','S0147824','S0147825','S0147826','S0147827','S0147828','S0147829','S0147830','S0147831','S0147832','S0147833','S0147834','S0147835','S0147836','S0147837','S0147838','S0147839','S0147840','S0147841','S0147842','S0147843','S0147844','S0147845','S0147846','S0147847','S0147848','S0147849','S0147850','S0147851','S0147852','S0147853','S0147854','S0147855','S0147856','S0147857','S0147858','S0147859','S0147860','S0147861','S0147862','S0147863','S0147864','S0147865','S0147866','S0147867','S0147868','S0147869','S0147870','S0147871','S0147872','S0147873','S0147874','S0147875','S0147876','S0147877','S0147878','S0147879','S0147880','S0147881','S0147882','S0147883','S0147884','S0147885','S0147886','S0147887','S0147888','S0147889','S0147890','S0147891','S0147892','S0147893','S0147894','S0147895','S0147896','S0147897','S0147898','S0147899','S0147900','S0147901','S0147902','S0147903','S0147904','S0147905','S0147906','S0147907','S0147908','S0147909','S0147910','S0147911','S0147912','S0147913','S0147914','S0147915','S0147916','S0147917','S0147918','S0147919','S0147920','S0147921','S0147922','S0147923','S0147924','S0147925','S0147926','S0147927','S0147928','S0147929','S0147930','S0147931','S0147932','S0147933','S0147934','S0147935','S0147936','S0147937','S0147938','S0147939','S0147940','S0147941','S0147942','S0147943','S0147944','S0147945','S0147946','S0147947','S0147948','S0147949','S0147950','S0147951','S0147952','S0147953','S0147954','S0147955','S0147956','S0147957','S0147958','S0147959','S0147960','S0147961','S0147962','S0147963','S0147964','S0147965','S0147966','S0147967','S0147968','S0147969','S0147970','S0147971','S0147972','S0147973','S0147974','S0147975','S0147976','S0147977','S0147978','S0147979','S0147980','S0147981','S0147982','S0147983','S0147984','S0147985','S0148056','S0148057','S0148058','S0148059','S0148060','S0148061','S0148062','S0148063','S0148064','S0148065','S0148066','S0148067','S0148068','S0148069','S0148070','S0148071','S0148072','S0148073','S0148074','S0148075','S0148076','S0148077','S0148078','S0148079','S0148080','S0148081','S0148082','S0148083','S0148084','S0148085','S0148086','S0148087','S0148088','S0148089','S0148090','S0148091','S0148092','S0148093','S0148094','S0148095','S0148096','S0148097','S0148098','S0148099','S0148100','S0148101','S0148102','S0148103','S0148104','S0148105','S0148106','S0148107','S0148108','S0148109','S0148110','S0148111','S0148112','S0148113','S0148114','S0148115','S0148116','S0148117','S0148118','S0148119','S0148120','S0148121','S0148122','S0148123','S0148124','S0148125','S0148126','S0148127','S0148128','S0148129','S0148130','S0148131','S0148132','S0148133','S0148134','S0148135','S0148136','S0148137','S0148138','S0148139','S0148140','S0148141','S0148142','S0148143','S0148144','S0148145','S0148146','S0148147','S0148148','S0148149','S0148150','S0148151','S0148152','S0148153','S0148154','S0148155','S0148156','S0148157','S0148158','S0148159','S0148160','S0148161','S0148162','S0148163','S0148164','S0148165','S0148166','S0148167','S0148168','S0148169','S0148170','S0148171','S0148172','S0148173','S0148174','S0148175','S0148176','S0148177','S0148178','S0148179','S0148180','S0148181','S0148182','S0148183','S0148184','S0148185','S0148186','S0148187','S0148188','S0148189','S0148190','S0148191','S0148192','S0148193','S0148194','S0148195','S0148196','S0148197','S0148198','S0148199','S0148200','S0148201','S0148202','S0148203','S0148204','S0148205','S0148206','S0148207','S0148208','S0148209','S0148210','S0148211','S0148212','S0148213','S0148214','S0148215','S0148216','S0148217','S0148218','S0148219','S0148220','S0148221','S0148222','S0148223','S0148224','S0148225','S0148226','S0148227','S0148228','S0148229','S0148230','S0148231','S0148232','S0148233','S0148234','S0148235','S0148236','S0148237','S0148238','S0148239','S0148240','S0148241','S0148242','S0148243','S0148244','S0148245','S0148246','S0148247','S0148248','S0148249','S0148250','S0148251','S0148252','S0148253','S0148254','S0148255','S0148256','S0148257','S0148258','S0148259','S0148260','S0148261','S0148262','S0148263','S0148264','S0148265','S0148266','S0148267','S0148268','S0148269','S0148270','S0148271','S0148272','S0148273','S0148274','S0148275','S0148276','S0148277','S0148278','S0148279','S0148280','S0148281','S0148282','S0148283','S0148284','S0148285','S0148286','S0148287','S0148288','S0148289','S0148290','S0148291','S0148292','S0148293','S0148294','S0148295','S0148296','S0148297','S0148298','S0148299','S0148300','S0148301','S0148302','S0148303','S0148304','S0148305','S0148306','S0148307','S0148308','S0148309','S0148310','S0148311','S0148312','S0148313','S0148314','S0148315','S0148316','S0148317','S0148318','S0148319','S0148320','S0148321','S0148322','S0148323','S0148324','S0148325','S0148326','S0148327','S0148328','S0148329','S0148330','S0148331','S0148332','S0148333','S0148334','S0148335','S0148336','S0148337','S0148338','S0148339','S0148340','S0148341','S0148342','S0148343','S0148344','S0148345','S0148346','S0148347','S0148348','S0148349','S0148350','S0148351','S0148352','S0148353','S0148354','S0148355','S0148356','S0148357','S0148358','S0148359','S0148360','S0148361','S0148362','S0148363','S0148364','S0148365','S0148366','S0148367','S0148368','S0148369','S0148370','S0148371','S0148372','S0148373','S0148374','S0148375','S0148376','S0148377','S0148378','S0148379','S0148380','S0148381','S0148382','S0148383','S0148384','S0148385','S0148386','S0148387','S0148388','S0148389','S0148390','S0148391','S0148392','S0148393','S0148394','S0148395','S0148396','S0148397','S0148398','S0148399','S0148400','S0148401','S0148402','S0148403','S0148404','S0148405','S0148406','S0148407','S0148408','S0148409','S0148410','S0148411','S0148412','S0148413','S0148414','S0148415','S0148416','S0148417','S0148418','S0148419','S0148420','S0148421','S0148422','S0148423','S0148424','S0148425','S0148426','S0148427','S0148428','S0148429','S0148430','S0148431','S0148432','S0148433','S0148434','S0148435','S0148436','S0148437','S0148438','S0148439','S0148440','S0148441','S0148442','S0148443','S0148444','S0148445','S0148446','S0148447','S0148448','S0148449','S0148450','S0148451','S0148452','S0148453','S0148454','S0148455','S0148456','S0148457','S0148458','S0148459','S0148460','S0148461','S0148462','S0148463','S0148464','S0148465','S0148466','S0148467','S0148468','S0148469','S0148470','S0148471','S0148472','S0148473','S0148474','S0148475','S0148476','S0148477','S0148478','S0148479','S0148480','S0148481','S0148482','S0148483','S0148484','S0148485','S0148486','S0148487','S0148488','S0148489','S0148490','S0148491','S0148492','S0148493','S0148494','S0148495','S0148496','S0148497','S0148498','S0148499','S0148500','S0148501','S0148502','S0148503','S0148504','S0148505','S0148506','S0148507','S0148508','S0148509','S0148510','S0148511','S0148512','S0148513','S0148514','S0148515','S0148516','S0148517','S0148518','S0148519','S0148520','S0148521','S0148522','S0148523','S0148524','S0148525','S0148526','S0148527','S0148528','S0148529','S0148530','S0148531','S0148532','S0148533','S0148534','S0148535','S0148536','S0148537','S0148538','S0148539','S0148540','S0148541','S0148542','S0148543','S0148544','S0148545','S0148546','S0148547','S0148548','S0148549','S0148550','S0148551','S0148552','S0148553','S0148554','S0148555','S0148556','S0148557','S0148558','S0148559','S0148560','S0148561','S0148562','S0148563','S0148564','S0148565','S0148566','S0148567','S0148568','S0148569','S0148570','S0148571','S0148572','S0148573','S0148574','S0148575','S0148576','S0148577','S0148578','S0148579','S0148580','S0148581','S0148582','S0148583','S0148584','S0148585','S0148586','S0148587','S0148588','S0148589','S0148590','S0148591','S0148592','S0148593','S0148594','S0148595','S0148596','S0148597','S0148598','S0148599','S0148600','S0148601','S0148602','S0148603','S0148604','S0148605','S0148606','S0148607','S0148608','S0148609','S0148610','S0148611','S0148612','S0148613','S0148614','S0148615','S0148616','S0148617','S0148618','S0148619','S0148620','S0148621','S0148622','S0148623','S0148624','S0148625','S0148626','S0148627','S0148628','S0148629','S0148630','S0148631','S0148632','S0148633','S0148634','S0148635','S0148636','S0148637','S0148638','S0148639','S0148640','S0148641','S0148642','S0148643','S0148644','S0148645','S0148646','S0148647','S0148648','S0148649','S0148650','S0148651','S0148652','S0148653','S0148654','S0148655','S0148656','S0148657','S0148658','S0148659','S0148660','S0148661','S0148662','S0148663','S0148664','S0148665','S0148666','S0148667','S0148668','S0148669','S0148670','S0148671','S0148672','S0148673','S0148674','S0148675','S0148676','S0148677','S0148678','S0148679','S0148680','S0148681','S0148682','S0148683','S0148684','S0148685'],
 "cities": [{'code': '110100', 'name': '北京'}, {'code': '120100', 'name': '天津'}, {'code': '130100', 'name': '石家庄'}, {'code': '140100', 'name': '太原'}, {'code': '150100', 'name': '呼和浩特'}, {'code': '210100', 'name': '沈阳'}, {'code': '210200', 'name': '大连'}, {'code': '220100', 'name': '长春'}, {'code': '230100', 'name': '哈尔滨'}, {'code': '310100', 'name': '上海'}, {'code': '320100', 'name': '南京'}, {'code': '330100', 'name': '杭州'}, {'code': '330200', 'name': '宁波'}, {'code': '340100', 'name': '合肥'}, {'code': '350100', 'name': '福州'}, {'code': '350200', 'name': '厦门'}, {'code': '360100', 'name': '南昌'}, {'code': '370100', 'name': '济南'}, {'code': '370200', 'name': '青岛'}, {'code': '410100', 'name': '郑州'}, {'code': '420100', 'name': '武汉'}, {'code': '430100', 'name': '长沙'}, {'code': '440100', 'name': '广州'}, {'code': '440300', 'name': '深圳'}, {'code': '450100', 'name': '南宁'}, {'code': '460100', 'name': '海口'}, {'code': '510100', 'name': '成都'}, {'code': '520100', 'name': '贵阳'}, {'code': '530100', 'name': '昆明'}, {'code': '500100', 'name': '重庆'}, {'code': '610100', 'name': '西安'}, {'code': '620100', 'name': '兰州'}, {'code': '630100', 'name': '西宁'}, {'code': '640100', 'name': '银川'}, {'code': '650100', 'name': '乌鲁木齐'}, {'code': '130200', 'name': '唐山'}, {'code': '130300', 'name': '秦皇岛'}, {'code': '150200', 'name': '包头'}, {'code': '210600', 'name': '丹东'}, {'code': '210700', 'name': '锦州'}, {'code': '220200', 'name': '吉林'}, {'code': '231000', 'name': '牡丹江'}, {'code': '320200', 'name': '无 锡'}, {'code': '321000', 'name': '扬州'}, {'code': '320300', 'name': '徐州'}, {'code': '330300', 'name': '温州'}, {'code': '330700', 'name': '金华'}, {'code': '340300', 'name': '蚌埠'}, {'code': '340800', 'name': '安庆'}, {'code': '350500', 'name': '泉州'}, {'code': '360400', 'name': '九江'}, {'code': '360700', 'name': '赣州'}, {'code': '370600', 'name': '烟台'}, {'code': '370800', 'name': '济宁'}, {'code': '410300', 'name': '洛阳'}, {'code': '410400', 'name': '平顶山'}, {'code': '420500', 'name': '宜昌'}, {'code': '420600', 'name': '襄阳'}, {'code': '430600', 'name': '岳阳'}, {'code': '430700', 'name': '常德'}, {'code': '441300', 'name': '惠州'}, {'code': '440800', 'name': '湛江'}, {'code': '440200', 'name': '韶关'}, {'code': '450300', 'name': '桂林'}, {'code': '450500', 'name': '北海'}, {'code': '460200', 'name': '三亚'}, {'code': '510500', 'name': '泸州'}, {'code': '511300', 'name': '南充'}, {'code': '520300', 'name': '遵义'}, {'code': '532900', 'name': '大理'}],
 "output": "WDPYTHON_HOUSE_YYYYMMDD.txt"
 }
}

'''
 ['S0147356', 'S0147425'] -- 90平以下新建住宅当月同比
 ['S0147426', 'S0147495'] -- 90平以下新建住宅环比
 ['S0147496', 'S0147565'] -- 90平以下新建住宅定基
 ['S0147566', 'S0147635'] -- 90~144平新建住宅当月同比
 ['S0147636', 'S0147705'] -- 90~144平新建住宅环比
 ['S0147706', 'S0147775'] -- 90~144平新建住宅定基
 ['S0147776', 'S0147845'] -- 144平以上新建住宅当月同比
 ['S0147846', 'S0147915'] -- 144平以上新建住宅环比
 ['S0147916', 'S0147985'] -- 144平以上新建住宅定基

['S0148056', 'S0148125'] -- 90平以下二手房当月同比
 ['S0148126', 'S0148195'] -- 90平以下二手房环比
 ['S0148196', 'S0148265'] -- 90平以下二手房定基
 ['S0148266', 'S0148335'] -- 90~144平二手房当月同比
 ['S0148336', 'S0148405'] -- 90~144平二手房环比
 ['S0148406', 'S0148475'] -- 90~144平二手房定基
 ['S0148476', 'S0148545'] -- 144平以上二手房当月同比
 ['S0148546', 'S0148615'] -- 144平以上二手房环比
 ['S0148616', 'S0148685'] -- 144平以上二手房定基
'''
def exec_m_house(exec_date):
 codes = ','.join(fields_config['house']['codes'])
 date = datetime.datetime.strptime(exec_date, '%Y%m%d').strftime('%Y-%m-%d')
 try_times = 0
 ret = None
 while True:
 try_times = try_times + 1
 log('第{}次尝试获取edb数据'.format(str(try_times)))
 ret = w.edb(codes, date, date,"Fill=Previous")
 if ret.ErrorCode == 0:
 break
 time.sleep(5)
 if try_times >= 5:
 log('获取edb数据异常,请联系管理员')
 break
 data = ret.Data[0]
 cities_new = copy.deepcopy(fields_config['house']['cities'])
 cities_old = copy.deepcopy(fields_config['house']['cities'])
 output_name = fields_config['house']['output'].replace('YYYYMMDD', exec_date)

for i in range(70):
 cities_new[i]['90_tongbi'] = data[i]
 cities_new[i]['90_huanbi'] = data[i+70]
 cities_new[i]['90_dingji'] = data[i+140]

cities_new[i]['90_144_tongbi'] = data[i+210]
 cities_new[i]['90_144_huanbi'] = data[i+280]
 cities_new[i]['90_144_dingji'] = data[i+350]

cities_new[i]['144_tongbi'] = data[i+420]
 cities_new[i]['144_huanbi'] = data[i+490]
 cities_new[i]['144_dingji'] = data[i+560]

cities_old[i]['90_tongbi'] = data[i+630]
 cities_old[i]['90_huanbi'] = data[i+700]
 cities_old[i]['90_dingji'] = data[i+770]

cities_old[i]['90_144_tongbi'] = data[i+840]
 cities_old[i]['90_144_huanbi'] = data[i+910]
 cities_old[i]['90_144_dingji'] = data[i+980]

cities_old[i]['144_tongbi'] = data[i+1050]
 cities_old[i]['144_huanbi'] = data[i+1120]
 cities_old[i]['144_dingji'] = data[i+1190]

# 最终待写入数据
 rows = []
 for city in cities_new:
 row = [exec_date, city['code'], '100500', 
 str(city['90_tongbi']), str(city['90_144_tongbi']), str(city['144_tongbi']), 
 str(city['90_huanbi']), str(city['90_144_huanbi']), str(city['144_huanbi']),
 str(city['90_dingji']), str(city['90_144_dingji']), str(city['144_dingji'])]
 rows.append('|'.join(row))

for city in cities_old:
 row = [exec_date, city['code'], '100100', 
 str(city['90_tongbi']), str(city['90_144_tongbi']), str(city['144_tongbi']), 
 str(city['90_huanbi']), str(city['90_144_huanbi']), str(city['144_huanbi']),
 str(city['90_dingji']), str(city['90_144_dingji']), str(city['144_dingji'])]
 rows.append('|'.join(row))
 write_file(output_name, rows)

'''
 调用wind接口获取耵市数据
'''
def exec_m_data(exec_date):
 input_list = get_input(exec_date)
 if input_list == None:
 log('exec_m_data input is None')
 return
 for key in input_list.keys():
 t = input_list[key]['type']
 codes = input_list[key]['codes']
 date = input_list[key]['date']
 fields = fields_config[key]['fields']
 fields_len = fields_config[key]['len']
 output_name = fields_config[key]['output'].replace('YYYYMMDD', exec_date)
 try_times = 0
 ret = None
 while True:
 try_times = try_times + 1
 log('第{}次尝试获取wss数据'.format(str(try_times)))
 ret = w.wss(codes, fields, options='tradeDate={};credibility=1'.format(date))
 if ret.ErrorCode == 0:
 break
 time.sleep(5)
 if try_times >= 5:
 log('获取wss数据异常,请联系管理员')
 break
 data = ret.Data
 data_converted = data_convert(date, data)
 write_file(output_name, data_converted)
 time.sleep(4)

'''
 抓取债券信用评级数据
'''
def exec_rating_data(date):
 # 要抓取的债券列表
 codes = get_input_rating_codes(date)
 if codes == '' or codes == None:
 log('exec_rating_data input is empty')
 return
 # 信用评级的相关字段
 fields = fields_config['rating']['fields']
 # 调用wind接口
 try_times = 0
 ret = None
 while True:
 try_times = try_times + 1
 log('第{}次尝试获取edb数据'.format(str(try_times)))
 ret = w.wss(codes, fields)
 if ret.ErrorCode == 0:
 break
 time.sleep(5)
 if try_times >= 5:
 log('获取edb数据异常,请联系管理员')
 break
 data = ret.Data
 #数据转换
 data_converted = data_convert(date, data)
 #将记录写入文件
 output_name = fields_config['rating']['output'].replace('YYYYMMDD', date)
 write_file(output_name, data_converted)

'''
 数据格式转换公共方法
'''
def data_convert(date, data):
 # 记录数
 row_len = len(data[0])
 # 每条记录的字段数
 fields_len = len(data)
 # 转换后的数据
 rows = []
 for i in range(row_len):
 row = []
 row.append(date)
 for j in range(fields_len):
 value = data[j][i]
 if value == None:value = ''
 if isinstance(value, float) or isinstance(value, int):
 value = str(round(value,6))
 if isinstance(value, datetime.datetime):
 try:
 value = value.strftime('%y%m%d')
 except:
 value = '19000101'
 value = value.replace('|', '')
 row.append(value)
 if row[1] == '':
 log('skip blank code')
 continue
 log('|'.join(row))
 rows.append('|'.join(row))
 return rows

'''
 将记录写入文件
'''
def write_file(output_name, rows):
 local_abs_path = os.path.join(LOCAL_PATH, output_name)
 if os.path.exists(local_abs_path):
 timestamp = datetime.datetime.now().strftime('%y%m%d%I%M%S')
 os.rename(local_abs_path, os.path.join(local_path, 'bak_' + output_name + '_' + timestamp))
 f = open(local_abs_path, 'a')
 for line in rows:
 log(line)
 line += '\n'
 f.write(line)
 f.close()

'''
 读取配置文件获债券信用评级的输入证券列表
'''
def get_input_rating_codes(date):
 fname = os.path.join(LOCAL_PATH, "RWMS_BONDLIST_YYYYMMDD.txt".replace('YYYYMMDD', date))
 if not os.path.exists(fname):
 raise FileNotFoundError('未找到文件')

codes = [code2wincode(x.strip(), 'bond') for x in open(fname).readlines() if x.strip() != '']
 return codes

'''
 读取配置文件获取耵市数据的输入证券列表
'''
def get_input(date):
 fname = os.path.join(LOCAL_PATH, "CMS_MARK_TO_MARKET_YYYYMMDD.txt".replace('YYYYMMDD', date))
 if not os.path.exists(fname):
 raise FileNotFoundError('未找到配置文件')
 input_list = [x.strip().split('|') for x in open(fname).readlines() if x.strip() != '']
 ret = {}
 ret['stock'] = {}
 ret['stock']['type'] = 'stock'
 ret['stock']['codes'] = []
 ret['stock']['date'] = ''
 ret['fund'] = {}
 ret['fund']['type'] = 'fund'
 ret['fund']['codes'] = []
 ret['fund']['date'] = ''
 ret['bond'] = {}
 ret['bond']['type'] = 'bond'
 ret['bond']['codes'] = []
 ret['bond']['date'] = ''
 for item in input_list:
 #todo: 根据item[0],item[1]进行转换
 wincode = code2wincode(item[1], item[0])
 #ret[item[0]]['codes'].append(item[1])
 ret[item[0]]['codes'].append(wincode)
 ret[item[0]]['date'] = item[2]
 ret['stock']['codes'] = ','.join(ret['stock']['codes'])
 ret['fund']['codes'] = ','.join(ret['fund']['codes'])
 ret['bond']['codes'] = ','.join(ret['bond']['codes'])
 return ret

'''
 将不带后缀的证券编码根据类型转换为带后缀的证券编码
 举例: 1111111 --> 111111.SH
'''
def code2wincode(code, t):
 if code.find('.') >= 0:
 return code
 elif t == 'stock':
 return w.htocode(code, "stocka").Data[0][0]
 elif t == 'bond':
 return w.htocode(code, "bond").Data[0][0]
 elif t == 'fund':
 return w.htocode(code, "fund").Data[0][0]

def main(date):

# 下载输入文件
 util = FtpUtil(HOST, PORT, USER, PASSWORD, TIMEOUT)
 util.set_local_path(LOCAL_PATH)
 util.set_server_path(SERVER_PATH)
 files = ['CMS_MARK_TO_MARKET_YYYYMMDD.txt', 'RWMS_BONDLIST_YYYYMMDD.txt']
 files = [x.replace('YYYYMMDD', date) for x in files]
 util.downloads(files)

# 抓取耵市数据
 exec_m_data(date)
 # 抓取债券评级数据
 exec_rating_data(date)
 # 抓取房地产数据
 exec_m_house(date)

# 上传输出文件
 list = ['WDPYTHON_BOND_YYYYMMDD.txt', 
 'WDPYTHON_BOND_RATING_YYYYMMDD.txt', 
 'WDPYTHON_FUND_YYYYMMDD.txt', 
 'WDPYTHON_HOUSE_YYYYMMDD.txt', 
 'WDPYTHON_STOCK_YYYYMMDD.txt',]
 list = [ x.replace('YYYYMMDD', date) for x in list]
 for item in list:
 abs_local_path = os.path.join(LOCAL_PATH, item)
 if not os.path.exists(abs_local_path):
 log('{0} not created,please check.'.format(abs_local_path))
 continue
 util.upload(item)

if __name__ == '__main__':
 date = datetime.datetime.today().strftime("%Y%m%d")
 if len(sys.argv) == 2:
 date = sys.argv[1].strip()
 print(date)
 log(date)
 username = str(base64.b64decode(WIND_USERNAME), 'UTF-8')
 password = str(base64.b64decode(WIND_PASSWORD), 'UTF-8')
 w.start(username=username, password=password)
 time.sleep(5)
 main(date)
 #w.close()

一键复制编辑原始数据按行查看历史

内部项目提交于 2021年08月13日 16:44 +08:00 . Linux下调用wind Python API 获取股票、债券、基金耵市数据,以及70个中大城市新建商品房/二手住房房价数据.

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398

# -*- encoding=utf-8 -*-
from WindPy import *
import datetime
import os
import copy
import paramiko
import base64
import configparser
import time


config = configparser.ConfigParser()
config.read('wind.ini')
HOST = config['FTP']['host']
PORT = int(config['FTP']['port'])
USER = config['FTP']['user']
PASSWORD = config['FTP']['password']
TIMEOUT = int(config['FTP']['timeout'])
LOCAL_PATH = config['FTP']['local_path']
SERVER_PATH = config['FTP']['server_path']
WIND_USERNAME = config['WIND']['username']
WIND_PASSWORD = config['WIND']['password']
log_file = config['LOG']['log_file']

class FtpUtil:
 def __init__(self, host, port, user, passwd, timeout=10):
 self.server_path = ''
 self.local_path = ''
 _ = paramiko.Transport((host, port))
 _.banner_timeout = timeout
 _.connect(username=user, password=passwd)
 self.sftp = paramiko.SFTPClient.from_transport(_)

 def set_server_path(self, p):
 self.server_path = p

 def set_local_path(self, p):
 self.local_path = p
 
 def download(self, fname):
 server_path = self.server_path + '/' + fname
 local_path = os.path.join(self.local_path,fname)
 self.sftp.get(server_path, local_path)

 def downloads(self, l):
 for item in l:
 self.download(item)

 def upload(self, fname):
 server_path = self.server_path + '/' + fname
 local_path = os.path.join(self.local_path,fname)
 self.sftp.put(local_path, server_path)

def log(msg):
 timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(time.time()))
 thread_name = threading.current_thread().getName()
 f = open(log_file, 'a')
 f.write('[' + timestamp + '][' + thread_name + ']' + msg + '\n')
 f.close()



fields_config = {
 "stock": {
 "type": "stock",
 "fields":"windcode,sec_name,volume,amt,turn,open,high,low,pre_close",
 "output": "WDPYTHON_STOCK_YYYYMMDD.txt",
 "len": 9
 },
 "fund":{
 "type": "fund",
 "fields": "windcode,fund_fullname,name_official,fund_type,nav,fund_navcur,nav_acc,fund_benchmark",
 "output": "WDPYTHON_FUND_YYYYMMDD.txt",
 "len": 8
 },
 "bond":{
 "type": "bond",
 "fields": "windcode,fullname,sec_name,exch_city,windl1type,windl2type,chinabondl1type,chinabondl2type,shclearl1type,yield_cnbd,net_cnbd,dirty_cnbd,yield_csi1,net_csi1,dirty_csi1",
 "output": "WDPYTHON_BOND_YYYYMMDD.txt",
 "len": 15
 },
 "rating": {
 "type": "rating",
 "fields": "windcode,issuerupdated,comp_name,registernumber,organizationcode,trade_code,amount,rate_latest,rate_changesofrating,rate_style,latestissurercreditrating,latestissurercreditratingdate,latestissurercreditratingtype",
 "output": "WDPYTHON_BOND_RATING_YYYYMMDD.txt",
 "len":13
 },
 "house": {
 "type": "house",
 "codes": ['S0147356','S0147357','S0147358','S0147359','S0147360','S0147361','S0147362','S0147363','S0147364','S0147365','S0147366','S0147367','S0147368','S0147369','S0147370','S0147371','S0147372','S0147373','S0147374','S0147375','S0147376','S0147377','S0147378','S0147379','S0147380','S0147381','S0147382','S0147383','S0147384','S0147385','S0147386','S0147387','S0147388','S0147389','S0147390','S0147391','S0147392','S0147393','S0147394','S0147395','S0147396','S0147397','S0147398','S0147399','S0147400','S0147401','S0147402','S0147403','S0147404','S0147405','S0147406','S0147407','S0147408','S0147409','S0147410','S0147411','S0147412','S0147413','S0147414','S0147415','S0147416','S0147417','S0147418','S0147419','S0147420','S0147421','S0147422','S0147423','S0147424','S0147425','S0147426','S0147427','S0147428','S0147429','S0147430','S0147431','S0147432','S0147433','S0147434','S0147435','S0147436','S0147437','S0147438','S0147439','S0147440','S0147441','S0147442','S0147443','S0147444','S0147445','S0147446','S0147447','S0147448','S0147449','S0147450','S0147451','S0147452','S0147453','S0147454','S0147455','S0147456','S0147457','S0147458','S0147459','S0147460','S0147461','S0147462','S0147463','S0147464','S0147465','S0147466','S0147467','S0147468','S0147469','S0147470','S0147471','S0147472','S0147473','S0147474','S0147475','S0147476','S0147477','S0147478','S0147479','S0147480','S0147481','S0147482','S0147483','S0147484','S0147485','S0147486','S0147487','S0147488','S0147489','S0147490','S0147491','S0147492','S0147493','S0147494','S0147495','S0147496','S0147497','S0147498','S0147499','S0147500','S0147501','S0147502','S0147503','S0147504','S0147505','S0147506','S0147507','S0147508','S0147509','S0147510','S0147511','S0147512','S0147513','S0147514','S0147515','S0147516','S0147517','S0147518','S0147519','S0147520','S0147521','S0147522','S0147523','S0147524','S0147525','S0147526','S0147527','S0147528','S0147529','S0147530','S0147531','S0147532','S0147533','S0147534','S0147535','S0147536','S0147537','S0147538','S0147539','S0147540','S0147541','S0147542','S0147543','S0147544','S0147545','S0147546','S0147547','S0147548','S0147549','S0147550','S0147551','S0147552','S0147553','S0147554','S0147555','S0147556','S0147557','S0147558','S0147559','S0147560','S0147561','S0147562','S0147563','S0147564','S0147565','S0147566','S0147567','S0147568','S0147569','S0147570','S0147571','S0147572','S0147573','S0147574','S0147575','S0147576','S0147577','S0147578','S0147579','S0147580','S0147581','S0147582','S0147583','S0147584','S0147585','S0147586','S0147587','S0147588','S0147589','S0147590','S0147591','S0147592','S0147593','S0147594','S0147595','S0147596','S0147597','S0147598','S0147599','S0147600','S0147601','S0147602','S0147603','S0147604','S0147605','S0147606','S0147607','S0147608','S0147609','S0147610','S0147611','S0147612','S0147613','S0147614','S0147615','S0147616','S0147617','S0147618','S0147619','S0147620','S0147621','S0147622','S0147623','S0147624','S0147625','S0147626','S0147627','S0147628','S0147629','S0147630','S0147631','S0147632','S0147633','S0147634','S0147635','S0147636','S0147637','S0147638','S0147639','S0147640','S0147641','S0147642','S0147643','S0147644','S0147645','S0147646','S0147647','S0147648','S0147649','S0147650','S0147651','S0147652','S0147653','S0147654','S0147655','S0147656','S0147657','S0147658','S0147659','S0147660','S0147661','S0147662','S0147663','S0147664','S0147665','S0147666','S0147667','S0147668','S0147669','S0147670','S0147671','S0147672','S0147673','S0147674','S0147675','S0147676','S0147677','S0147678','S0147679','S0147680','S0147681','S0147682','S0147683','S0147684','S0147685','S0147686','S0147687','S0147688','S0147689','S0147690','S0147691','S0147692','S0147693','S0147694','S0147695','S0147696','S0147697','S0147698','S0147699','S0147700','S0147701','S0147702','S0147703','S0147704','S0147705','S0147706','S0147707','S0147708','S0147709','S0147710','S0147711','S0147712','S0147713','S0147714','S0147715','S0147716','S0147717','S0147718','S0147719','S0147720','S0147721','S0147722','S0147723','S0147724','S0147725','S0147726','S0147727','S0147728','S0147729','S0147730','S0147731','S0147732','S0147733','S0147734','S0147735','S0147736','S0147737','S0147738','S0147739','S0147740','S0147741','S0147742','S0147743','S0147744','S0147745','S0147746','S0147747','S0147748','S0147749','S0147750','S0147751','S0147752','S0147753','S0147754','S0147755','S0147756','S0147757','S0147758','S0147759','S0147760','S0147761','S0147762','S0147763','S0147764','S0147765','S0147766','S0147767','S0147768','S0147769','S0147770','S0147771','S0147772','S0147773','S0147774','S0147775','S0147776','S0147777','S0147778','S0147779','S0147780','S0147781','S0147782','S0147783','S0147784','S0147785','S0147786','S0147787','S0147788','S0147789','S0147790','S0147791','S0147792','S0147793','S0147794','S0147795','S0147796','S0147797','S0147798','S0147799','S0147800','S0147801','S0147802','S0147803','S0147804','S0147805','S0147806','S0147807','S0147808','S0147809','S0147810','S0147811','S0147812','S0147813','S0147814','S0147815','S0147816','S0147817','S0147818','S0147819','S0147820','S0147821','S0147822','S0147823','S0147824','S0147825','S0147826','S0147827','S0147828','S0147829','S0147830','S0147831','S0147832','S0147833','S0147834','S0147835','S0147836','S0147837','S0147838','S0147839','S0147840','S0147841','S0147842','S0147843','S0147844','S0147845','S0147846','S0147847','S0147848','S0147849','S0147850','S0147851','S0147852','S0147853','S0147854','S0147855','S0147856','S0147857','S0147858','S0147859','S0147860','S0147861','S0147862','S0147863','S0147864','S0147865','S0147866','S0147867','S0147868','S0147869','S0147870','S0147871','S0147872','S0147873','S0147874','S0147875','S0147876','S0147877','S0147878','S0147879','S0147880','S0147881','S0147882','S0147883','S0147884','S0147885','S0147886','S0147887','S0147888','S0147889','S0147890','S0147891','S0147892','S0147893','S0147894','S0147895','S0147896','S0147897','S0147898','S0147899','S0147900','S0147901','S0147902','S0147903','S0147904','S0147905','S0147906','S0147907','S0147908','S0147909','S0147910','S0147911','S0147912','S0147913','S0147914','S0147915','S0147916','S0147917','S0147918','S0147919','S0147920','S0147921','S0147922','S0147923','S0147924','S0147925','S0147926','S0147927','S0147928','S0147929','S0147930','S0147931','S0147932','S0147933','S0147934','S0147935','S0147936','S0147937','S0147938','S0147939','S0147940','S0147941','S0147942','S0147943','S0147944','S0147945','S0147946','S0147947','S0147948','S0147949','S0147950','S0147951','S0147952','S0147953','S0147954','S0147955','S0147956','S0147957','S0147958','S0147959','S0147960','S0147961','S0147962','S0147963','S0147964','S0147965','S0147966','S0147967','S0147968','S0147969','S0147970','S0147971','S0147972','S0147973','S0147974','S0147975','S0147976','S0147977','S0147978','S0147979','S0147980','S0147981','S0147982','S0147983','S0147984','S0147985','S0148056','S0148057','S0148058','S0148059','S0148060','S0148061','S0148062','S0148063','S0148064','S0148065','S0148066','S0148067','S0148068','S0148069','S0148070','S0148071','S0148072','S0148073','S0148074','S0148075','S0148076','S0148077','S0148078','S0148079','S0148080','S0148081','S0148082','S0148083','S0148084','S0148085','S0148086','S0148087','S0148088','S0148089','S0148090','S0148091','S0148092','S0148093','S0148094','S0148095','S0148096','S0148097','S0148098','S0148099','S0148100','S0148101','S0148102','S0148103','S0148104','S0148105','S0148106','S0148107','S0148108','S0148109','S0148110','S0148111','S0148112','S0148113','S0148114','S0148115','S0148116','S0148117','S0148118','S0148119','S0148120','S0148121','S0148122','S0148123','S0148124','S0148125','S0148126','S0148127','S0148128','S0148129','S0148130','S0148131','S0148132','S0148133','S0148134','S0148135','S0148136','S0148137','S0148138','S0148139','S0148140','S0148141','S0148142','S0148143','S0148144','S0148145','S0148146','S0148147','S0148148','S0148149','S0148150','S0148151','S0148152','S0148153','S0148154','S0148155','S0148156','S0148157','S0148158','S0148159','S0148160','S0148161','S0148162','S0148163','S0148164','S0148165','S0148166','S0148167','S0148168','S0148169','S0148170','S0148171','S0148172','S0148173','S0148174','S0148175','S0148176','S0148177','S0148178','S0148179','S0148180','S0148181','S0148182','S0148183','S0148184','S0148185','S0148186','S0148187','S0148188','S0148189','S0148190','S0148191','S0148192','S0148193','S0148194','S0148195','S0148196','S0148197','S0148198','S0148199','S0148200','S0148201','S0148202','S0148203','S0148204','S0148205','S0148206','S0148207','S0148208','S0148209','S0148210','S0148211','S0148212','S0148213','S0148214','S0148215','S0148216','S0148217','S0148218','S0148219','S0148220','S0148221','S0148222','S0148223','S0148224','S0148225','S0148226','S0148227','S0148228','S0148229','S0148230','S0148231','S0148232','S0148233','S0148234','S0148235','S0148236','S0148237','S0148238','S0148239','S0148240','S0148241','S0148242','S0148243','S0148244','S0148245','S0148246','S0148247','S0148248','S0148249','S0148250','S0148251','S0148252','S0148253','S0148254','S0148255','S0148256','S0148257','S0148258','S0148259','S0148260','S0148261','S0148262','S0148263','S0148264','S0148265','S0148266','S0148267','S0148268','S0148269','S0148270','S0148271','S0148272','S0148273','S0148274','S0148275','S0148276','S0148277','S0148278','S0148279','S0148280','S0148281','S0148282','S0148283','S0148284','S0148285','S0148286','S0148287','S0148288','S0148289','S0148290','S0148291','S0148292','S0148293','S0148294','S0148295','S0148296','S0148297','S0148298','S0148299','S0148300','S0148301','S0148302','S0148303','S0148304','S0148305','S0148306','S0148307','S0148308','S0148309','S0148310','S0148311','S0148312','S0148313','S0148314','S0148315','S0148316','S0148317','S0148318','S0148319','S0148320','S0148321','S0148322','S0148323','S0148324','S0148325','S0148326','S0148327','S0148328','S0148329','S0148330','S0148331','S0148332','S0148333','S0148334','S0148335','S0148336','S0148337','S0148338','S0148339','S0148340','S0148341','S0148342','S0148343','S0148344','S0148345','S0148346','S0148347','S0148348','S0148349','S0148350','S0148351','S0148352','S0148353','S0148354','S0148355','S0148356','S0148357','S0148358','S0148359','S0148360','S0148361','S0148362','S0148363','S0148364','S0148365','S0148366','S0148367','S0148368','S0148369','S0148370','S0148371','S0148372','S0148373','S0148374','S0148375','S0148376','S0148377','S0148378','S0148379','S0148380','S0148381','S0148382','S0148383','S0148384','S0148385','S0148386','S0148387','S0148388','S0148389','S0148390','S0148391','S0148392','S0148393','S0148394','S0148395','S0148396','S0148397','S0148398','S0148399','S0148400','S0148401','S0148402','S0148403','S0148404','S0148405','S0148406','S0148407','S0148408','S0148409','S0148410','S0148411','S0148412','S0148413','S0148414','S0148415','S0148416','S0148417','S0148418','S0148419','S0148420','S0148421','S0148422','S0148423','S0148424','S0148425','S0148426','S0148427','S0148428','S0148429','S0148430','S0148431','S0148432','S0148433','S0148434','S0148435','S0148436','S0148437','S0148438','S0148439','S0148440','S0148441','S0148442','S0148443','S0148444','S0148445','S0148446','S0148447','S0148448','S0148449','S0148450','S0148451','S0148452','S0148453','S0148454','S0148455','S0148456','S0148457','S0148458','S0148459','S0148460','S0148461','S0148462','S0148463','S0148464','S0148465','S0148466','S0148467','S0148468','S0148469','S0148470','S0148471','S0148472','S0148473','S0148474','S0148475','S0148476','S0148477','S0148478','S0148479','S0148480','S0148481','S0148482','S0148483','S0148484','S0148485','S0148486','S0148487','S0148488','S0148489','S0148490','S0148491','S0148492','S0148493','S0148494','S0148495','S0148496','S0148497','S0148498','S0148499','S0148500','S0148501','S0148502','S0148503','S0148504','S0148505','S0148506','S0148507','S0148508','S0148509','S0148510','S0148511','S0148512','S0148513','S0148514','S0148515','S0148516','S0148517','S0148518','S0148519','S0148520','S0148521','S0148522','S0148523','S0148524','S0148525','S0148526','S0148527','S0148528','S0148529','S0148530','S0148531','S0148532','S0148533','S0148534','S0148535','S0148536','S0148537','S0148538','S0148539','S0148540','S0148541','S0148542','S0148543','S0148544','S0148545','S0148546','S0148547','S0148548','S0148549','S0148550','S0148551','S0148552','S0148553','S0148554','S0148555','S0148556','S0148557','S0148558','S0148559','S0148560','S0148561','S0148562','S0148563','S0148564','S0148565','S0148566','S0148567','S0148568','S0148569','S0148570','S0148571','S0148572','S0148573','S0148574','S0148575','S0148576','S0148577','S0148578','S0148579','S0148580','S0148581','S0148582','S0148583','S0148584','S0148585','S0148586','S0148587','S0148588','S0148589','S0148590','S0148591','S0148592','S0148593','S0148594','S0148595','S0148596','S0148597','S0148598','S0148599','S0148600','S0148601','S0148602','S0148603','S0148604','S0148605','S0148606','S0148607','S0148608','S0148609','S0148610','S0148611','S0148612','S0148613','S0148614','S0148615','S0148616','S0148617','S0148618','S0148619','S0148620','S0148621','S0148622','S0148623','S0148624','S0148625','S0148626','S0148627','S0148628','S0148629','S0148630','S0148631','S0148632','S0148633','S0148634','S0148635','S0148636','S0148637','S0148638','S0148639','S0148640','S0148641','S0148642','S0148643','S0148644','S0148645','S0148646','S0148647','S0148648','S0148649','S0148650','S0148651','S0148652','S0148653','S0148654','S0148655','S0148656','S0148657','S0148658','S0148659','S0148660','S0148661','S0148662','S0148663','S0148664','S0148665','S0148666','S0148667','S0148668','S0148669','S0148670','S0148671','S0148672','S0148673','S0148674','S0148675','S0148676','S0148677','S0148678','S0148679','S0148680','S0148681','S0148682','S0148683','S0148684','S0148685'],
 "cities": [{'code': '110100', 'name': '北京'}, {'code': '120100', 'name': '天津'}, {'code': '130100', 'name': '石家庄'}, {'code': '140100', 'name': '太原'}, {'code': '150100', 'name': '呼和浩特'}, {'code': '210100', 'name': '沈阳'}, {'code': '210200', 'name': '大连'}, {'code': '220100', 'name': '长春'}, {'code': '230100', 'name': '哈尔滨'}, {'code': '310100', 'name': '上海'}, {'code': '320100', 'name': '南京'}, {'code': '330100', 'name': '杭州'}, {'code': '330200', 'name': '宁波'}, {'code': '340100', 'name': '合肥'}, {'code': '350100', 'name': '福州'}, {'code': '350200', 'name': '厦门'}, {'code': '360100', 'name': '南昌'}, {'code': '370100', 'name': '济南'}, {'code': '370200', 'name': '青岛'}, {'code': '410100', 'name': '郑州'}, {'code': '420100', 'name': '武汉'}, {'code': '430100', 'name': '长沙'}, {'code': '440100', 'name': '广州'}, {'code': '440300', 'name': '深圳'}, {'code': '450100', 'name': '南宁'}, {'code': '460100', 'name': '海口'}, {'code': '510100', 'name': '成都'}, {'code': '520100', 'name': '贵阳'}, {'code': '530100', 'name': '昆明'}, {'code': '500100', 'name': '重庆'}, {'code': '610100', 'name': '西安'}, {'code': '620100', 'name': '兰州'}, {'code': '630100', 'name': '西宁'}, {'code': '640100', 'name': '银川'}, {'code': '650100', 'name': '乌鲁木齐'}, {'code': '130200', 'name': '唐山'}, {'code': '130300', 'name': '秦皇岛'}, {'code': '150200', 'name': '包头'}, {'code': '210600', 'name': '丹东'}, {'code': '210700', 'name': '锦州'}, {'code': '220200', 'name': '吉林'}, {'code': '231000', 'name': '牡丹江'}, {'code': '320200', 'name': '无 锡'}, {'code': '321000', 'name': '扬州'}, {'code': '320300', 'name': '徐州'}, {'code': '330300', 'name': '温州'}, {'code': '330700', 'name': '金华'}, {'code': '340300', 'name': '蚌埠'}, {'code': '340800', 'name': '安庆'}, {'code': '350500', 'name': '泉州'}, {'code': '360400', 'name': '九江'}, {'code': '360700', 'name': '赣州'}, {'code': '370600', 'name': '烟台'}, {'code': '370800', 'name': '济宁'}, {'code': '410300', 'name': '洛阳'}, {'code': '410400', 'name': '平顶山'}, {'code': '420500', 'name': '宜昌'}, {'code': '420600', 'name': '襄阳'}, {'code': '430600', 'name': '岳阳'}, {'code': '430700', 'name': '常德'}, {'code': '441300', 'name': '惠州'}, {'code': '440800', 'name': '湛江'}, {'code': '440200', 'name': '韶关'}, {'code': '450300', 'name': '桂林'}, {'code': '450500', 'name': '北海'}, {'code': '460200', 'name': '三亚'}, {'code': '510500', 'name': '泸州'}, {'code': '511300', 'name': '南充'}, {'code': '520300', 'name': '遵义'}, {'code': '532900', 'name': '大理'}],
 "output": "WDPYTHON_HOUSE_YYYYMMDD.txt"
 }
}


'''
 ['S0147356', 'S0147425'] -- 90平以下新建住宅当月同比
 ['S0147426', 'S0147495'] -- 90平以下新建住宅环比
 ['S0147496', 'S0147565'] -- 90平以下新建住宅定基
 ['S0147566', 'S0147635'] -- 90~144平新建住宅当月同比
 ['S0147636', 'S0147705'] -- 90~144平新建住宅环比
 ['S0147706', 'S0147775'] -- 90~144平新建住宅定基
 ['S0147776', 'S0147845'] -- 144平以上新建住宅当月同比
 ['S0147846', 'S0147915'] -- 144平以上新建住宅环比
 ['S0147916', 'S0147985'] -- 144平以上新建住宅定基

 ['S0148056', 'S0148125'] -- 90平以下二手房当月同比
 ['S0148126', 'S0148195'] -- 90平以下二手房环比
 ['S0148196', 'S0148265'] -- 90平以下二手房定基
 ['S0148266', 'S0148335'] -- 90~144平二手房当月同比
 ['S0148336', 'S0148405'] -- 90~144平二手房环比
 ['S0148406', 'S0148475'] -- 90~144平二手房定基
 ['S0148476', 'S0148545'] -- 144平以上二手房当月同比
 ['S0148546', 'S0148615'] -- 144平以上二手房环比
 ['S0148616', 'S0148685'] -- 144平以上二手房定基
'''
def exec_m_house(exec_date):
 codes = ','.join(fields_config['house']['codes'])
 date = datetime.datetime.strptime(exec_date, '%Y%m%d').strftime('%Y-%m-%d')
 try_times = 0
 ret = None
 while True:
 try_times = try_times + 1
 log('第{}次尝试获取edb数据'.format(str(try_times)))
 ret = w.edb(codes, date, date,"Fill=Previous")
 if ret.ErrorCode == 0:
 break
 time.sleep(5)
 if try_times >= 5:
 log('获取edb数据异常,请联系管理员')
 break
 data = ret.Data[0]
 cities_new = copy.deepcopy(fields_config['house']['cities'])
 cities_old = copy.deepcopy(fields_config['house']['cities'])
 output_name = fields_config['house']['output'].replace('YYYYMMDD', exec_date)

 for i in range(70):
 cities_new[i]['90_tongbi'] = data[i]
 cities_new[i]['90_huanbi'] = data[i+70]
 cities_new[i]['90_dingji'] = data[i+140]

 cities_new[i]['90_144_tongbi'] = data[i+210]
 cities_new[i]['90_144_huanbi'] = data[i+280]
 cities_new[i]['90_144_dingji'] = data[i+350]

 cities_new[i]['144_tongbi'] = data[i+420]
 cities_new[i]['144_huanbi'] = data[i+490]
 cities_new[i]['144_dingji'] = data[i+560]

 cities_old[i]['90_tongbi'] = data[i+630]
 cities_old[i]['90_huanbi'] = data[i+700]
 cities_old[i]['90_dingji'] = data[i+770]

 cities_old[i]['90_144_tongbi'] = data[i+840]
 cities_old[i]['90_144_huanbi'] = data[i+910]
 cities_old[i]['90_144_dingji'] = data[i+980]

 cities_old[i]['144_tongbi'] = data[i+1050]
 cities_old[i]['144_huanbi'] = data[i+1120]
 cities_old[i]['144_dingji'] = data[i+1190]

 # 最终待写入数据
 rows = []
 for city in cities_new:
 row = [exec_date, city['code'], '100500', 
 str(city['90_tongbi']), str(city['90_144_tongbi']), str(city['144_tongbi']), 
 str(city['90_huanbi']), str(city['90_144_huanbi']), str(city['144_huanbi']),
 str(city['90_dingji']), str(city['90_144_dingji']), str(city['144_dingji'])]
 rows.append('|'.join(row))

 for city in cities_old:
 row = [exec_date, city['code'], '100100', 
 str(city['90_tongbi']), str(city['90_144_tongbi']), str(city['144_tongbi']), 
 str(city['90_huanbi']), str(city['90_144_huanbi']), str(city['144_huanbi']),
 str(city['90_dingji']), str(city['90_144_dingji']), str(city['144_dingji'])]
 rows.append('|'.join(row))
 write_file(output_name, rows)


'''
 调用wind接口获取耵市数据
'''
def exec_m_data(exec_date):
 input_list = get_input(exec_date)
 if input_list == None:
 log('exec_m_data input is None')
 return
 for key in input_list.keys():
 t = input_list[key]['type']
 codes = input_list[key]['codes']
 date = input_list[key]['date']
 fields = fields_config[key]['fields']
 fields_len = fields_config[key]['len']
 output_name = fields_config[key]['output'].replace('YYYYMMDD', exec_date)
 try_times = 0
 ret = None
 while True:
 try_times = try_times + 1
 log('第{}次尝试获取wss数据'.format(str(try_times)))
 ret = w.wss(codes, fields, options='tradeDate={};credibility=1'.format(date))
 if ret.ErrorCode == 0:
 break
 time.sleep(5)
 if try_times >= 5:
 log('获取wss数据异常,请联系管理员')
 break
 data = ret.Data
 data_converted = data_convert(date, data)
 write_file(output_name, data_converted)
 time.sleep(4)


'''
 抓取债券信用评级数据
'''
def exec_rating_data(date):
 # 要抓取的债券列表
 codes = get_input_rating_codes(date)
 if codes == '' or codes == None:
 log('exec_rating_data input is empty')
 return
 # 信用评级的相关字段
 fields = fields_config['rating']['fields']
 # 调用wind接口
 try_times = 0
 ret = None
 while True:
 try_times = try_times + 1
 log('第{}次尝试获取edb数据'.format(str(try_times)))
 ret = w.wss(codes, fields)
 if ret.ErrorCode == 0:
 break
 time.sleep(5)
 if try_times >= 5:
 log('获取edb数据异常,请联系管理员')
 break
 data = ret.Data
 #数据转换
 data_converted = data_convert(date, data)
 #将记录写入文件
 output_name = fields_config['rating']['output'].replace('YYYYMMDD', date)
 write_file(output_name, data_converted)


'''
 数据格式转换公共方法
'''
def data_convert(date, data):
 # 记录数
 row_len = len(data[0])
 # 每条记录的字段数
 fields_len = len(data)
 # 转换后的数据
 rows = []
 for i in range(row_len):
 row = []
 row.append(date)
 for j in range(fields_len):
 value = data[j][i]
 if value == None:value = ''
 if isinstance(value, float) or isinstance(value, int):
 value = str(round(value,6))
 if isinstance(value, datetime.datetime):
 try:
 value = value.strftime('%y%m%d')
 except:
 value = '19000101'
 value = value.replace('|', '')
 row.append(value)
 if row[1] == '':
 log('skip blank code')
 continue
 log('|'.join(row))
 rows.append('|'.join(row))
 return rows


'''
 将记录写入文件
'''
def write_file(output_name, rows):
 local_abs_path = os.path.join(LOCAL_PATH, output_name)
 if os.path.exists(local_abs_path):
 timestamp = datetime.datetime.now().strftime('%y%m%d%I%M%S')
 os.rename(local_abs_path, os.path.join(local_path, 'bak_' + output_name + '_' + timestamp))
 f = open(local_abs_path, 'a')
 for line in rows:
 log(line)
 line += '\n'
 f.write(line)
 f.close()


'''
 读取配置文件获债券信用评级的输入证券列表
'''
def get_input_rating_codes(date):
 fname = os.path.join(LOCAL_PATH, "RWMS_BONDLIST_YYYYMMDD.txt".replace('YYYYMMDD', date))
 if not os.path.exists(fname):
 raise FileNotFoundError('未找到文件')

 codes = [code2wincode(x.strip(), 'bond') for x in open(fname).readlines() if x.strip() != '']
 return codes


'''
 读取配置文件获取耵市数据的输入证券列表
'''
def get_input(date):
 fname = os.path.join(LOCAL_PATH, "CMS_MARK_TO_MARKET_YYYYMMDD.txt".replace('YYYYMMDD', date))
 if not os.path.exists(fname):
 raise FileNotFoundError('未找到配置文件')
 input_list = [x.strip().split('|') for x in open(fname).readlines() if x.strip() != '']
 ret = {}
 ret['stock'] = {}
 ret['stock']['type'] = 'stock'
 ret['stock']['codes'] = []
 ret['stock']['date'] = ''
 ret['fund'] = {}
 ret['fund']['type'] = 'fund'
 ret['fund']['codes'] = []
 ret['fund']['date'] = ''
 ret['bond'] = {}
 ret['bond']['type'] = 'bond'
 ret['bond']['codes'] = []
 ret['bond']['date'] = ''
 for item in input_list:
 #todo: 根据item[0],item[1]进行转换
 wincode = code2wincode(item[1], item[0])
 #ret[item[0]]['codes'].append(item[1])
 ret[item[0]]['codes'].append(wincode)
 ret[item[0]]['date'] = item[2]
 ret['stock']['codes'] = ','.join(ret['stock']['codes'])
 ret['fund']['codes'] = ','.join(ret['fund']['codes'])
 ret['bond']['codes'] = ','.join(ret['bond']['codes'])
 return ret


'''
 将不带后缀的证券编码根据类型转换为带后缀的证券编码
 举例: 1111111 --> 111111.SH
'''
def code2wincode(code, t):
 if code.find('.') >= 0:
 return code
 elif t == 'stock':
 return w.htocode(code, "stocka").Data[0][0]
 elif t == 'bond':
 return w.htocode(code, "bond").Data[0][0]
 elif t == 'fund':
 return w.htocode(code, "fund").Data[0][0]


def main(date):

 # 下载输入文件
 util = FtpUtil(HOST, PORT, USER, PASSWORD, TIMEOUT)
 util.set_local_path(LOCAL_PATH)
 util.set_server_path(SERVER_PATH)
 files = ['CMS_MARK_TO_MARKET_YYYYMMDD.txt', 'RWMS_BONDLIST_YYYYMMDD.txt']
 files = [x.replace('YYYYMMDD', date) for x in files]
 util.downloads(files)

 # 抓取耵市数据
 exec_m_data(date)
 # 抓取债券评级数据
 exec_rating_data(date)
 # 抓取房地产数据
 exec_m_house(date)

 # 上传输出文件
 list = ['WDPYTHON_BOND_YYYYMMDD.txt', 
 'WDPYTHON_BOND_RATING_YYYYMMDD.txt', 
 'WDPYTHON_FUND_YYYYMMDD.txt', 
 'WDPYTHON_HOUSE_YYYYMMDD.txt', 
 'WDPYTHON_STOCK_YYYYMMDD.txt',]
 list = [ x.replace('YYYYMMDD', date) for x in list]
 for item in list:
 abs_local_path = os.path.join(LOCAL_PATH, item)
 if not os.path.exists(abs_local_path):
 log('{0} not created,please check.'.format(abs_local_path))
 continue
 util.upload(item)


if __name__ == '__main__':
 date = datetime.datetime.today().strftime("%Y%m%d")
 if len(sys.argv) == 2:
 date = sys.argv[1].strip()
 print(date)
 log(date)
 username = str(base64.b64decode(WIND_USERNAME), 'UTF-8')
 password = str(base64.b64decode(WIND_PASSWORD), 'UTF-8')
 w.start(username=username, password=password)
 time.sleep(5)
 main(date)
 #w.close()

误判申诉

此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。

如您确认内容无涉及不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。

取消

提交

简介

个人AI助理,音频实时监听,GPT对话,opencv获取摄像头视频截图;Python基础编程示例:Excel读写追加处理,XML解析、JSON解析、FLV与MP4转换,PyQT界面应用程序开发示例等,https证书到期检测,糗百爬虫,pdf和图片互相转换,socket使用,百度OCR调用例子,IP及端口快速扫描。

暂无标签

https://gitee.com/mktime

使用 MIT 开源许可协议

0 Stars

1 Watching

265 Forks

取消