# -*- encoding:utf-8 -*-
from __future__ import print_function
from __future__ import division

import warnings
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import seaborn as sns

# noinspection PyUnresolvedReferences
import abu_local_env
import abupy
from abupy import ABuSymbolPd
from abupy import xrange, pd_resample

warnings.filterwarnings('ignore')
sns.set_context(rc={'figure.figsize': (14, 7)})
# 使用沙盒数据,目的是和书中一样的数据环境
abupy.env.enable_example_env_ipython()

stock_day_change = np.load('../gen/stock_day_change.npy')


"""
 第四章 量化工具——pandas

 abu量化系统github地址:https://github.com/bbfamily/abu (您的star是我的动力!)
 abu量化文档教程ipython notebook:https://github.com/bbfamily/abu/tree/master/abupy_lecture
"""


def sample_411():
 """
 4.1.1 DataFrame构建及方法
 :return:
 """
 print('stock_day_change.shape:', stock_day_change.shape)

 # 下面三种写法输出完全相同,输出如表4-1所示
 print('head():\n', pd.DataFrame(stock_day_change).head())
 print('head(5):\n', pd.DataFrame(stock_day_change).head(5))
 print('[:5]:\n', pd.DataFrame(stock_day_change)[:5])


def sample_412():
 """
 4.1.2 索引行列序列
 :return:
 """
 # 股票0 -> 股票stock_day_change.shape[0]
 stock_symbols = ['股票 ' + str(x) for x in
 xrange(stock_day_change.shape[0])]
 # 通过构造直接设置index参数,head(2)就显示两行,表4-2所示
 print('pd.DataFrame(stock_day_change, index=stock_symbols).head(2):\n',
 pd.DataFrame(stock_day_change, index=stock_symbols).head(2))
 # 从2017年1月1日向上时间递进,单位freq='1d'即1天
 days = pd.date_range('2017-1-1',
 periods=stock_day_change.shape[1], freq='1d')
 # 股票0 -> 股票stock_day_change.shape[0]
 stock_symbols = ['股票 ' + str(x) for x in
 xrange(stock_day_change.shape[0])]
 # 分别设置index和columns
 df = pd.DataFrame(stock_day_change, index=stock_symbols, columns=days)
 # 表4-3所示
 print('df.head(2):\n', df.head(2))


def sample_413():
 """
 4.1.3 金融时间序列
 :return:
 """
 days = pd.date_range('2017-1-1',
 periods=stock_day_change.shape[1], freq='1d')
 stock_symbols = ['股票 ' + str(x) for x in
 xrange(stock_day_change.shape[0])]
 df = pd.DataFrame(stock_day_change, index=stock_symbols, columns=days)

 # df做个转置
 df = df.T
 # 表4-4所示
 print('df.head():\n', df.head())

 df_20 = pd_resample(df, '21D', how='mean')
 # 表4-5所示
 print('df_20.head():\n', df_20.head())


def sample_414():
 """
 4.1.4 Series构建及方法
 :return
 """
 days = pd.date_range('2017-1-1',
 periods=stock_day_change.shape[1], freq='1d')
 stock_symbols = ['股票 ' + str(x) for x in
 xrange(stock_day_change.shape[0])]
 df = pd.DataFrame(stock_day_change, index=stock_symbols, columns=days)
 df = df.T

 print('df.head():\n', df.head())
 df_stock0 = df['股票 0']
 # 打印df_stock0类型
 print('type(df_stock0):', type(df_stock0))
 # 打印出Series的前5行数据, 与DataFrame一致
 print('df_stock0.head():\n', df_stock0.head())

 df_stock0.cumsum().plot()
 plt.show()


def sample_415():
 """
 4.1.5 重采样数据
 :return
 """
 days = pd.date_range('2017-1-1',
 periods=stock_day_change.shape[1], freq='1d')
 stock_symbols = ['股票 ' + str(x) for x in
 xrange(stock_day_change.shape[0])]
 df = pd.DataFrame(stock_day_change, index=stock_symbols, columns=days)
 df = df.T
 df_stock0 = df['股票 0']

 # 以5天为周期重采样(周k)
 df_stock0_5 = pd_resample(df_stock0.cumsum(), '5D', how='ohlc')
 # 以21天为周期重采样(月k),
 # noinspection PyUnusedLocal
 df_stock0_20 = pd_resample(df_stock0.cumsum(), '21D', how='ohlc')
 # 打印5天重采样,如下输出2017年01月01日, 2017年01月06日, 2017年01月11日, 表4-6所示
 print('df_stock0_5.head():\n', df_stock0_5.head())

 from abupy import ABuMarketDrawing
 # 图4-2所示
 ABuMarketDrawing.plot_candle_stick(df_stock0_5.index,
 df_stock0_5['open'].values,
 df_stock0_5['high'].values,
 df_stock0_5['low'].values,
 df_stock0_5['close'].values,
 np.random.random(len(df_stock0_5)),
 None, 'stock', day_sum=False,
 html_bk=False, save=False)

 print('type(df_stock0_5.open.values):', type(df_stock0_5['open'].values))
 print('df_stock0_5.open.index:\n', df_stock0_5['open'].index)
 print('df_stock0_5.columns:\n', df_stock0_5.columns)


"""
 4.2 基本数据分析示例
"""
# n_folds=2两年
tsla_df = ABuSymbolPd.make_kl_df('usTSLA', n_folds=2)


def sample_420():
 # 表4-7所示
 print('tsla_df.tail():\n', tsla_df.tail())


def sample_421():
 """
 4.2.1 数据整体分析
 :return:
 """
 print('tsla_df.info():\n', tsla_df.info())
 print('tsla_df.describe():\n', tsla_df.describe())

 tsla_df[['close', 'volume']].plot(subplots=True, style=['r', 'g'], grid=True)
 plt.show()


def sample_422():
 """
 4.2.2 索引选取和切片选择
 :return:
 """

 # 2014年07月23日至2014年07月31日 开盘价格序列
 print('tsla_df.loc[x:x, x]\n', tsla_df.loc['2014-07-23':'2014-07-31', 'open'])

 # 2014年07月23日至2014年07月31日 所有序列,表4-9所示
 print('tsla_df.loc[x:x]\n', tsla_df.loc['2014-07-23':'2014-07-31'])

 # [1:5]:(1,2,3,4),[2:6]: (2, 3, 4, 5)
 # 表4-10所示
 print('tsla_df.iloc[1:5, 2:6]:\n', tsla_df.iloc[1:5, 2:6])

 # 切取所有行[2:6]: (2, 3, 4, 5)列
 print('tsla_df.iloc[:, 2:6]:\n', tsla_df.iloc[:, 2:6])
 # 选取所有的列[35:37]:(35, 36)行,表4-11所示
 print('tsla_df.iloc[35:37]:\n', tsla_df.iloc[35:37])

 # 指定一个列
 print('tsla_df.close[0:3]:\n', tsla_df.close[0:3])
 # 通过组成一个列表选择多个列,表4-12所示
 print('tsla_df[][0:3]:\n', tsla_df[['close', 'high', 'low']][0:3])


def sample_423():
 """
 4.2.3 逻辑条件进行数据筛选
 :return:
 """
 # abs为取绝对值的意思,不是防抱死,表4-13所示
 print('tsla_df[np.abs(tsla_df.p_change) > 8]:\n', tsla_df[np.abs(tsla_df.p_change) > 8])
 print('tsla_df[(np.abs(tsla_df.p_change) > 8) & (tsla_df.volume > 2.5 * tsla_df.volume.mean())]:\n',
 tsla_df[(np.abs(tsla_df.p_change) > 8) & (tsla_df.volume > 2.5 * tsla_df.volume.mean())])


def sample_424_1():
 """
 4.2.4_1 数据转换与规整
 :return:
 """
 # 数据序列值排序
 print('tsla_df.sort_index(by=p_change)[:5]:\n', tsla_df.sort_index(by='p_change')[:5])
 print('tsla_df.sort_index(by=p_change, ascending=False)[:5]:\n',
 tsla_df.sort_index(by='p_change', ascending=False)[:5])

 # 如果一行的数据中存在na就删除这行
 tsla_df.dropna()
 # 通过how控制 如果一行的数据中全部都是na就删除这行
 tsla_df.dropna(how='all')
 # 使用指定值填充na, inplace代表就地操作,即不返回新的序列在原始序列上修改
 tsla_df.fillna(tsla_df.mean(), inplace=True)


def sample_424_2():
 """
 4.2.4_1 数据转换处理 pct_change
 :return:
 """
 print('tsla_df.close[:3]:\n', tsla_df.close[:3])
 print('tsla_df.close.pct_change()[:3]:\n', tsla_df.close.pct_change()[:3])
 print('(223.54 - 222.49) / 222.49, (223.57 - 223.54) / 223.54:', (223.54 - 222.49) / 222.49,
 (223.57 - 223.54) / 223.54)

 # pct_change对序列从第二项开始向前做减法在除以前一项,这样的针对close做pct_change后的结果就是涨跌幅
 change_ratio = tsla_df.close.pct_change()
 print('change_ratio.tail():\n', change_ratio.tail())

 # 将change_ratio转变成与tsla_df.p_change字段一样的百分百,同样保留两位小数
 print('np.round(change_ratio[-5:] * 100, 2):\n', np.round(change_ratio[-5:] * 100, 2))

 fmt = lambda x: '%.2f' % x
 print('tsla_df.atr21.map(fmt).tail():\n', tsla_df.atr21.map(fmt).tail())


def sample_425():
 """
 4.2.5 数据本地序列化操作
 :return:
 """
 tsla_df.to_csv('../gen/tsla_df.csv', columns=tsla_df.columns, index=True)
 tsla_df_load = pd.read_csv('../gen/tsla_df.csv', parse_dates=True, index_col=0)
 print('tsla_df_load.head():\n', tsla_df_load.head())


"""
 4.3 实例1:寻找股票异动涨跌幅阀值
"""


def sample_431():
 """
 4.3.1 数据的离散化
 :return:
 """
 tsla_df.p_change.hist(bins=80)
 plt.show()

 cats = pd.qcut(np.abs(tsla_df.p_change), 10)
 print('cats.value_counts():\n', cats.value_counts())

 # 将涨跌幅数据手工分类,从负无穷到-7,-5,-3,0, 3, 5, 7,正无穷
 bins = [-np.inf, -7.0, -5, -3, 0, 3, 5, 7, np.inf]
 cats = pd.cut(tsla_df.p_change, bins)
 print('bins cats.value_counts():\n', cats.value_counts())

 # cr_dummies为列名称前缀
 change_ration_dummies = pd.get_dummies(cats, prefix='cr_dummies')
 print('change_ration_dummies.head():\n', change_ration_dummies.head())


def sample_432():
 """
 4.3.2 concat, append, merge的使用
 :return:
 """
 # 将涨跌幅数据手工分类,从负无穷到-7,-5,-3,0, 3, 5, 7,正无穷
 bins = [-np.inf, -7.0, -5, -3, 0, 3, 5, 7, np.inf]
 cats = pd.cut(tsla_df.p_change, bins)
 change_ration_dummies = pd.get_dummies(cats, prefix='cr_dummies')

 # noinspection PyUnresolvedReferences
 print('pd.concat([tsla_df, change_ration_dummies], axis=1).tail():\n ',
 pd.concat([tsla_df, change_ration_dummies], axis=1).tail())

 # pd.concat的连接axis=0:纵向连接atr>14的df和p_change > 10的df
 pd.concat([tsla_df[tsla_df.p_change > 10],
 tsla_df[tsla_df.atr14 > 16]], axis=0)

 # 直接使用DataFrame对象append,结果与上面pd.concat的结果一致, 表4-20所示
 print('tsla_df[tsla_df.p_change > 10].append(tsla_df[tsla_df.atr14 > 16]):\n',
 tsla_df[tsla_df.p_change > 10].append(tsla_df[tsla_df.atr14 > 16]))


"""
 4.4 实例2 :星期几是这个股票的‘好日子’
"""


def sample_441():
 """
 4.4.1 构建交叉表
 :return:
 """
 # noinspection PyTypeChecker
 tsla_df['positive'] = np.where(tsla_df.p_change > 0, 1, 0)
 print('tsla_df.tail():\n', tsla_df.tail())
 xt = pd.crosstab(tsla_df.date_week, tsla_df.positive)
 print('xt:\n', xt)

 xt_pct = xt.div(xt.sum(1).astype(float), axis=0)
 print('xt_pct:\n', xt_pct)

 xt_pct.plot(
 figsize=(8, 5),
 kind='bar',
 stacked=True,
 title='date_week -> positive')
 plt.xlabel('date_week')
 plt.ylabel('positive')
 plt.show()


def sample_442():
 """
 4.4.2 构建透视表
 :return:
 """
 # noinspection PyTypeChecker
 tsla_df['positive'] = np.where(tsla_df.p_change > 0, 1, 0)
 print('tsla_df.pivot_table([positive], index=[date_week]):\n',
 tsla_df.pivot_table(['positive'], index=['date_week']))
 print('tsla_df.groupby([date_week, positive])[positive].count():\n',
 tsla_df.groupby(['date_week', 'positive'])['positive'].count())


"""
 4.5 实例3 :跳空缺口
"""

jump_pd = pd.DataFrame()
jump_threshold = tsla_df.close.median() * 0.03


def judge_jump(p_today):
 global jump_pd
 if p_today.p_change > 0 and (p_today.low - p_today.pre_close) > jump_threshold:
 """
 符合向上跳空
 """
 # jump记录方向 1向上
 p_today['jump'] = 1
 # 向上跳能量=(今天最低 - 昨收)/ 跳空阀值
 p_today['jump_power'] = (p_today.low - p_today.pre_close) / jump_threshold
 jump_pd = jump_pd.append(p_today)
 elif p_today.p_change < 0 and (p_today.pre_close - p_today.high) > jump_threshold:
 """
 符合向下跳空
 """
 # jump记录方向 -1向下
 p_today['jump'] = -1
 # 向下跳能量=(昨收 - 今天最高)/ 跳空阀值
 p_today['jump_power'] = (p_today.pre_close - p_today.high) / jump_threshold
 jump_pd = jump_pd.append(p_today)


def sample_45_1():
 """
 4.5 实例3 :跳空缺口
 :return:
 """
 for kl_index in np.arange(0, tsla_df.shape[0]):
 # 通过ix一个一个拿
 today = tsla_df.ix[kl_index]
 judge_jump(today)

 # filter按照顺序只显示这些列, 表4-26所示
 print('jump_pd.filter([jump, jump_power, close, date, p_change, pre_close]):\n',
 jump_pd.filter(['jump', 'jump_power', 'close', 'date', 'p_change', 'pre_close']))


def sample_45_2():
 """
 4.5 实例3 :跳空缺口
 :return:
 """
 # axis=1即行数据,tsla_df的每一条行数据即为每一个交易日数据
 tsla_df.apply(judge_jump, axis=1)
 print('jump_pd:\n', jump_pd)

 from abupy import ABuMarketDrawing
 # view_indexs传入jump_pd.index,即在k图上使用圆来标示跳空点
 ABuMarketDrawing.plot_candle_form_klpd(tsla_df, view_indexs=jump_pd.index)
 plt.show()


"""
 4.6 pandas三维面板的使用
"""


def sample_46():
 """
 4.6 pandas三维面板的使用
 :return:
 """
 # disable_example_env_ipython不再使用沙盒数据,因为沙盒里面没有相关tsla行业的数据啊
 abupy.env.disable_example_env_ipython()

 from abupy import ABuIndustries
 r_symbol = 'usTSLA'
 # 这里获取了和TSLA电动车处于同一行业的股票组成pandas三维面板Panel数据
 p_date, _ = ABuIndustries.get_industries_panel_from_target(r_symbol, show=False)
 print('type(p_date):', type(p_date))
 print('p_date:\n', p_date)

 print('p_date[usTTM].head():\n', p_date['usTTM'].head())

 p_data_it = p_date.swapaxes('items', 'minor')
 print('p_data_it:\n', p_data_it)

 p_data_it_close = p_data_it['close'].dropna(axis=0)
 print('p_data_it_close.tail():\n', p_data_it_close.tail())

 from abupy import ABuScalerUtil
 # ABuScalerUtil.scaler_std将所有close的切面数据做(group - group.mean()) / group.std()标示化,为了可视化在同一范围
 p_data_it_close = ABuScalerUtil.scaler_std(p_data_it_close)
 p_data_it_close.plot()
 plt.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)
 plt.ylabel('Price')
 plt.xlabel('Time')
 plt.show()


if __name__ == "__main__":
 sample_411()
 # sample_412()
 # sample_413()
 # sample_414()
 # sample_415()
 # sample_420()
 # sample_421()
 # sample_422()
 # sample_423()
 # sample_424_1()
 # sample_424_2()
 # sample_425()
 # sample_431()
 # sample_432()
 # sample_441()
 # sample_442()
 # sample_45_1()
 # sample_45_2()
 # sample_46()