968 lines
36 KiB
Python
968 lines
36 KiB
Python
# -*- encoding:utf-8 -*-
|
||
from __future__ import print_function
|
||
import matplotlib.pyplot as plt
|
||
import seaborn as sns
|
||
import numpy as np
|
||
import pandas as pd
|
||
# import warnings
|
||
|
||
# noinspection PyUnresolvedReferences
|
||
import abu_local_env
|
||
import abupy
|
||
from abupy import abu
|
||
from abupy import ABuSymbolPd
|
||
|
||
import sklearn.preprocessing as preprocessing
|
||
|
||
# warnings.filterwarnings('ignore')
|
||
sns.set_context(rc={'figure.figsize': (14, 7)})
|
||
# 使用沙盒数据，目的是和书中一样的数据环境
|
||
abupy.env.enable_example_env_ipython()
|
||
|
||
"""
|
||
第10章 量化系统——机器学习•猪老三
|
||
|
||
abu量化系统github地址：https://github.com/bbfamily/abu (您的star是我的动力！)
|
||
abu量化文档教程ipython notebook：https://github.com/bbfamily/abu/tree/master/abupy_lecture
|
||
"""
|
||
|
||
"""
|
||
10.2 猪老三世界中的量化环境
|
||
"""
|
||
|
||
"""
|
||
是否开启date_week噪音, 开启这个的目的是让分类结果正确率降低，接近真实
|
||
"""
|
||
g_with_date_week_noise = False
|
||
|
||
|
||
def _gen_another_word_price(kl_another_word):
|
||
"""
|
||
生成股票在另一个世界中的价格
|
||
:param kl_another_word:
|
||
:return:
|
||
"""
|
||
for ind in np.arange(2, kl_another_word.shape[0]):
|
||
# 前天数据
|
||
bf_yesterday = kl_another_word.iloc[ind - 2]
|
||
# 昨天
|
||
yesterday = kl_another_word.iloc[ind - 1]
|
||
# 今天
|
||
today = kl_another_word.iloc[ind]
|
||
# 生成今天的收盘价格
|
||
kl_another_word.close[ind] = _gen_another_word_price_rule(
|
||
yesterday.close, yesterday.volume,
|
||
bf_yesterday.close, bf_yesterday.volume,
|
||
today.volume, today.date_week)
|
||
|
||
|
||
def _gen_another_word_price_rule(yesterday_close, yesterday_volume,
|
||
bf_yesterday_close,
|
||
bf_yesterday_volume,
|
||
today_volume, date_week):
|
||
"""
|
||
通过前天收盘量价，昨天收盘量价，今天的量，构建另一个世界中的价格模型
|
||
"""
|
||
# 昨天收盘价格与前天收盘价格的价格差
|
||
price_change = yesterday_close - bf_yesterday_close
|
||
# 昨天成交量与前天成交量的量差
|
||
volume_change = yesterday_volume - bf_yesterday_volume
|
||
|
||
# 如果量和价变动一致，今天价格涨，否则跌
|
||
# 即量价齐涨－>涨, 量价齐跌－>涨，量价不一致－>跌
|
||
sign = 1.0 if price_change * volume_change > 0 else -1.0
|
||
|
||
# 通过date_week生成噪音，否则之后分类100%分对
|
||
if g_with_date_week_noise:
|
||
# 针对sign生成噪音，噪音的生效的先决条件是今天的量是这三天最大的
|
||
gen_noise = today_volume > np.max(
|
||
[yesterday_volume, bf_yesterday_volume])
|
||
# 如果量是这三天最大 且是周五，下跌
|
||
if gen_noise and date_week == 4:
|
||
sign = -1.0
|
||
# 如果量是这三天最大，如果是周一，上涨
|
||
elif gen_noise and date_week == 0:
|
||
sign = 1.0
|
||
|
||
# 今天的涨跌幅度基础是price_change（昨天前天的价格变动）
|
||
price_base = abs(price_change)
|
||
# 今天的涨跌幅度变动因素：量比，
|
||
# 今天的成交量/昨天的成交量 和 今天的成交量/前天的成交量 的均值
|
||
price_factor = np.mean([today_volume / yesterday_volume,
|
||
today_volume / bf_yesterday_volume])
|
||
|
||
if abs(price_base * price_factor) < yesterday_close * 0.10:
|
||
# 如果 量比 * price_base 没超过10%，今天价格计算
|
||
today_price = yesterday_close + \
|
||
sign * price_base * price_factor
|
||
else:
|
||
# 如果涨跌幅度超过10%，限制上限，下限为10%
|
||
today_price = yesterday_close + sign * yesterday_close * 0.10
|
||
return today_price
|
||
|
||
|
||
def change_real_to_another_word(symbol):
|
||
"""
|
||
将原始真正的股票数据价格列只保留前两天数据，成交量，周几列完全保留
|
||
价格列其他数据使用_gen_another_word_price变成另一个世界价格
|
||
:param symbol:
|
||
:return:
|
||
"""
|
||
kl_pd = ABuSymbolPd.make_kl_df(symbol)
|
||
if kl_pd is not None:
|
||
# 原始股票数据也只保留价格，周几，成交量
|
||
kl_pig_three = kl_pd.filter(['close', 'date_week', 'volume'])
|
||
# 只保留原始头两天的交易收盘价格，其他的的都赋予nan
|
||
kl_pig_three['close'][2:] = np.nan
|
||
# 将其他nan价格变成猪老三世界中价格使用_gen_another_word_price
|
||
_gen_another_word_price(kl_pig_three)
|
||
return kl_pig_three
|
||
|
||
|
||
def sample_102(show=True):
|
||
"""
|
||
10.2 生成猪老三的世界中的映射股票数据
|
||
:return:
|
||
"""
|
||
choice_symbols = ['usNOAH', 'usSFUN', 'usBIDU', 'usAAPL', 'usGOOG',
|
||
'usTSLA', 'usWUBA', 'usVIPS']
|
||
another_word_dict = {}
|
||
real_dict = {}
|
||
for symbol in choice_symbols:
|
||
# 猪老三世界的股票走势字典
|
||
another_word_dict[symbol] = change_real_to_another_word(symbol)
|
||
# 真实世界的股票走势字典，这里不考虑运行效率问题
|
||
real_dict[symbol] = ABuSymbolPd.make_kl_df(symbol)
|
||
if show:
|
||
# 表10-1所示
|
||
print('another_word_dict[usNOAH].head():\n', another_word_dict['usNOAH'].head())
|
||
|
||
print('real_dict[usNOAH].head():\n', real_dict['usNOAH'].head().filter(['close', 'date_week', 'volume']))
|
||
|
||
import itertools
|
||
# 4 ＊ 2
|
||
_, axs = plt.subplots(nrows=4, ncols=2, figsize=(20, 15))
|
||
# 将画布序列拉平
|
||
axs_list = list(itertools.chain.from_iterable(axs))
|
||
|
||
for symbol, ax in zip(choice_symbols, axs_list):
|
||
# 绘制猪老三世界的股价走势
|
||
another_word_dict[symbol].close.plot(ax=ax)
|
||
# 同样的股票在真实世界的股价走势
|
||
real_dict[symbol].close.plot(ax=ax)
|
||
ax.set_title(symbol)
|
||
plt.show()
|
||
return another_word_dict
|
||
|
||
|
||
"""
|
||
10.3 有监督机器学习
|
||
"""
|
||
|
||
|
||
def gen_pig_three_feature(kl_another_word):
|
||
"""
|
||
猪老三构建特征模型函数
|
||
:param kl_another_word: 即上一节使用_gen_another_word_price
|
||
生成的dataframe有收盘价，周几，成交量列
|
||
:return:
|
||
"""
|
||
# y值使用close.pct_change即涨跌幅度
|
||
kl_another_word['regress_y'] = kl_another_word.close.pct_change()
|
||
# 前天收盘价格
|
||
kl_another_word['bf_yesterday_close'] = 0
|
||
# 昨天收盘价格
|
||
kl_another_word['yesterday_close'] = 0
|
||
# 昨天收盘成交量
|
||
kl_another_word['yesterday_volume'] = 0
|
||
# 前天收盘成交量
|
||
kl_another_word['bf_yesterday_volume'] = 0
|
||
|
||
# 对齐特征，前天收盘价格即与今天的收盘错2个时间单位，[2:] = [:-2]
|
||
kl_another_word['bf_yesterday_close'][2:] = \
|
||
kl_another_word['close'][:-2]
|
||
# 对齐特征，前天成交量
|
||
kl_another_word['bf_yesterday_volume'][2:] = \
|
||
kl_another_word['volume'][:-2]
|
||
# 对齐特征，昨天收盘价与今天的收盘错1个时间单位，[1:] = [:-1]
|
||
kl_another_word['yesterday_close'][1:] = \
|
||
kl_another_word['close'][:-1]
|
||
# 对齐特征，昨天成交量
|
||
kl_another_word['yesterday_volume'][1:] = \
|
||
kl_another_word['volume'][:-1]
|
||
|
||
# 特征1: 价格差
|
||
kl_another_word['feature_price_change'] = \
|
||
kl_another_word['yesterday_close'] - \
|
||
kl_another_word['bf_yesterday_close']
|
||
|
||
# 特征2: 成交量差
|
||
kl_another_word['feature_volume_Change'] = \
|
||
kl_another_word['yesterday_volume'] - \
|
||
kl_another_word['bf_yesterday_volume']
|
||
|
||
# 特征3: 涨跌sign
|
||
kl_another_word['feature_sign'] = np.sign(
|
||
kl_another_word['feature_price_change'] * kl_another_word[
|
||
'feature_volume_Change'])
|
||
|
||
# 特征4: 周几
|
||
kl_another_word['feature_date_week'] = kl_another_word[
|
||
'date_week']
|
||
|
||
"""
|
||
构建噪音特征, 因为猪老三也不可能全部分析正确真实的特征因素
|
||
这里引入一些噪音特征
|
||
"""
|
||
# 成交量乘积
|
||
kl_another_word['feature_volume_noise'] = \
|
||
kl_another_word['yesterday_volume'] * \
|
||
kl_another_word['bf_yesterday_volume']
|
||
|
||
# 价格乘积
|
||
kl_another_word['feature_price_noise'] = \
|
||
kl_another_word['yesterday_close'] * \
|
||
kl_another_word['bf_yesterday_close']
|
||
|
||
# 将数据标准化
|
||
scaler = preprocessing.StandardScaler()
|
||
kl_another_word['feature_price_change'] = scaler.fit_transform(
|
||
kl_another_word['feature_price_change'].values.reshape(-1, 1))
|
||
kl_another_word['feature_volume_Change'] = scaler.fit_transform(
|
||
kl_another_word['feature_volume_Change'].values.reshape(-1, 1))
|
||
kl_another_word['feature_volume_noise'] = scaler.fit_transform(
|
||
kl_another_word['feature_volume_noise'].values.reshape(-1, 1))
|
||
kl_another_word['feature_price_noise'] = scaler.fit_transform(
|
||
kl_another_word['feature_price_noise'].values.reshape(-1, 1))
|
||
|
||
# 只筛选feature_开头的特征和regress_y，抛弃前两天数据，即[2:]
|
||
kl_pig_three_feature = kl_another_word.filter(
|
||
regex='regress_y|feature_*')[2:]
|
||
return kl_pig_three_feature
|
||
|
||
|
||
def sample_103_0(show=True):
|
||
"""
|
||
10.3 生成猪老三的训练集特征示例
|
||
:return:
|
||
"""
|
||
another_word_dict = sample_102(show=False)
|
||
pig_three_feature = None
|
||
for symbol in another_word_dict:
|
||
# 首先拿出对应的走势数据
|
||
kl_another_word = another_word_dict[symbol]
|
||
# 通过走势数据生成训练集特征通过gen_pig_three_feature
|
||
kl_feature = gen_pig_three_feature(kl_another_word)
|
||
# 将每个股票的特征数据都拼接起来，形成训练集
|
||
pig_three_feature = kl_feature if pig_three_feature is None \
|
||
else pig_three_feature.append(kl_feature)
|
||
|
||
# Dataframe -> matrix
|
||
feature_np = pig_three_feature.as_matrix()
|
||
# x特征矩阵
|
||
train_x = feature_np[:, 1:]
|
||
# 回归训练的连续值y
|
||
train_y_regress = feature_np[:, 0]
|
||
# 分类训练的离散值y，之后分类技术使用
|
||
# noinspection PyTypeChecker
|
||
train_y_classification = np.where(train_y_regress > 0, 1, 0)
|
||
|
||
if show:
|
||
print('pig_three_feature.shape:', pig_three_feature.shape)
|
||
print('pig_three_feature.tail():\n', pig_three_feature.tail())
|
||
print('train_x[:5], train_y_regress[:5], train_y_classification[:5]:\n', train_x[:5], train_y_regress[:5],
|
||
train_y_classification[:5])
|
||
|
||
return train_x, train_y_regress, train_y_classification, pig_three_feature
|
||
|
||
|
||
"""
|
||
猪老三使用回归预测股价
|
||
"""
|
||
|
||
|
||
def sample_1031_1():
|
||
"""
|
||
10.3.1_1 猪老三使用回归预测股价：生成训练集数据和测试集数据
|
||
:return:
|
||
"""
|
||
|
||
# noinspection PyShadowingNames
|
||
def gen_feature_from_symbol(symbol):
|
||
"""
|
||
封装由一个symbol转换为特征矩阵序列函数
|
||
:param symbol:
|
||
:return:
|
||
"""
|
||
# 真实世界走势数据转换到老三的世界
|
||
kl_another_word = change_real_to_another_word(symbol)
|
||
# 由走势转换为特征dataframe通过gen_pig_three_feature
|
||
kl_another_word_feature_test = gen_pig_three_feature(kl_another_word)
|
||
# 转换为matrix
|
||
feature_np_test = kl_another_word_feature_test.as_matrix()
|
||
# 从matrix抽取y回归
|
||
test_y_regress = feature_np_test[:, 0]
|
||
# y回归 －> y分类
|
||
# noinspection PyTypeChecker
|
||
test_y_classification = np.where(test_y_regress > 0, 1, 0)
|
||
# 从matrix抽取x特征矩阵
|
||
test_x = feature_np_test[:, 1:]
|
||
return test_x, test_y_regress, test_y_classification, kl_another_word_feature_test
|
||
|
||
# 生成训练集数据
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature = sample_103_0(show=False)
|
||
# 生成测试集数据
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = gen_feature_from_symbol('usFB')
|
||
|
||
print('训练集：{}, 测试集：{}'.format(pig_three_feature.shape[0], kl_another_word_feature_test.shape[0]))
|
||
|
||
return train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test
|
||
|
||
|
||
def regress_process(estimator, train_x, train_y_regress, test_x,
|
||
test_y_regress):
|
||
# 训练训练集数据
|
||
estimator.fit(train_x, train_y_regress)
|
||
# 使用训练好的模型预测测试集对应的y，即根据usFB的走势特征预测股价涨跌幅度
|
||
test_y_prdict_regress = estimator.predict(test_x)
|
||
|
||
# 绘制usFB实际股价涨跌幅度
|
||
plt.plot(test_y_regress.cumsum())
|
||
# 绘制通过模型预测的usFB股价涨跌幅度
|
||
plt.plot(test_y_prdict_regress.cumsum())
|
||
|
||
# 针对训练集数据做交叉验证
|
||
from abupy import cross_val_score
|
||
from abupy.CoreBu.ABuFixes import mean_squared_error_scorer
|
||
scores = cross_val_score(estimator, train_x,
|
||
train_y_regress, cv=10,
|
||
scoring=mean_squared_error_scorer)
|
||
# mse开方 -> rmse
|
||
mean_sc = -np.mean(np.sqrt(-scores))
|
||
print('{} RMSE: {}'.format(estimator.__class__.__name__, mean_sc))
|
||
|
||
|
||
def sample_1031_2():
|
||
"""
|
||
10.3.1_2 猪老三使用回归预测股价：LinearRegressio
|
||
:return:
|
||
"""
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
# 实例化线性回归对象estimator
|
||
from sklearn.linear_model import LinearRegression
|
||
estimator = LinearRegression()
|
||
# 将回归模型对象，训练集x，训练集连续y值，测试集x，测试集连续y传入
|
||
regress_process(estimator, train_x, train_y_regress, test_x,
|
||
test_y_regress)
|
||
plt.show()
|
||
|
||
from abupy import ABuMLExecute
|
||
ABuMLExecute.plot_learning_curve(estimator, train_x, train_y_regress, cv=10)
|
||
|
||
|
||
def sample_1031_3():
|
||
"""
|
||
10.3.1_3 猪老三使用回归预测股价：PolynomialFeatures
|
||
:return:
|
||
"""
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
from sklearn.pipeline import make_pipeline
|
||
from sklearn.preprocessing import PolynomialFeatures
|
||
from sklearn.linear_model import LinearRegression
|
||
|
||
# pipeline套上 degree=3 ＋ LinearRegression
|
||
estimator = make_pipeline(PolynomialFeatures(degree=3),
|
||
LinearRegression())
|
||
# 继续使用regress_process，区别是estimator变了
|
||
regress_process(estimator, train_x, train_y_regress, test_x,
|
||
test_y_regress)
|
||
plt.show()
|
||
|
||
|
||
def sample_1031_4():
|
||
"""
|
||
10.3.1_4 猪老三使用回归预测股价：使用集成学习算法预测股价AdaBoost与RandomForest
|
||
:return:
|
||
"""
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
# AdaBoost
|
||
from sklearn.ensemble import AdaBoostRegressor
|
||
|
||
estimator = AdaBoostRegressor(n_estimators=100)
|
||
regress_process(estimator, train_x, train_y_regress, test_x,
|
||
test_y_regress)
|
||
plt.show()
|
||
# RandomForest
|
||
from sklearn.ensemble import RandomForestRegressor
|
||
|
||
estimator = RandomForestRegressor(n_estimators=100)
|
||
regress_process(estimator, train_x, train_y_regress, test_x, test_y_regress)
|
||
plt.show()
|
||
|
||
|
||
"""
|
||
10.3.2 猪老三使用分类预测股票涨跌
|
||
"""
|
||
|
||
|
||
def classification_process(estimator, train_x, train_y_classification,
|
||
test_x, test_y_classification):
|
||
from sklearn import metrics
|
||
# 训练数据，这里分类要所以要使用y_classification
|
||
estimator.fit(train_x, train_y_classification)
|
||
# 使用训练好的分类模型预测测试集对应的y，即根据usFB的走势特征预测涨跌
|
||
test_y_prdict_classification = estimator.predict(test_x)
|
||
# 通过metrics.accuracy_score度量预测涨跌的准确率
|
||
print("{} accuracy = {:.2f}".format(
|
||
estimator.__class__.__name__,
|
||
metrics.accuracy_score(test_y_classification,
|
||
test_y_prdict_classification)))
|
||
|
||
from abupy import cross_val_score
|
||
# 针对训练集数据做交叉验证scoring='accuracy'，cv＝10
|
||
scores = cross_val_score(estimator, train_x,
|
||
train_y_classification,
|
||
cv=10,
|
||
scoring='accuracy')
|
||
# 所有交叉验证的分数取平均值
|
||
mean_sc = np.mean(scores)
|
||
print('cross validation accuracy mean: {:.2f}'.format(mean_sc))
|
||
|
||
|
||
def sample_1032_1():
|
||
"""
|
||
10.3.2_1 猪老三使用分类预测股票涨跌：LogisticRegression
|
||
:return:
|
||
"""
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
# 无噪音分类正确100%
|
||
from sklearn.linear_model import LogisticRegression
|
||
estimator = LogisticRegression(C=1.0, penalty='l1', tol=1e-6)
|
||
# 将分类器，训练集x，训练集y分类，测试集，测试集y分别传入函数
|
||
classification_process(estimator, train_x, train_y_classification,
|
||
test_x, test_y_classification)
|
||
|
||
# 开启噪音，再来一遍，有噪音正确率93%, 之后的都开启g_with_date_week_noise
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
classification_process(estimator, train_x, train_y_classification,
|
||
test_x, test_y_classification)
|
||
|
||
|
||
def sample_1032_2():
|
||
"""
|
||
10.3.2_2 猪老三使用分类预测股票涨跌：svm
|
||
:return:
|
||
"""
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
from sklearn.svm import SVC
|
||
|
||
estimator = SVC(kernel='rbf')
|
||
classification_process(estimator, train_x, train_y_classification,
|
||
test_x, test_y_classification)
|
||
|
||
|
||
def sample_1032_3():
|
||
"""
|
||
10.3.2_3 猪老三使用分类预测股票涨跌：RandomForestClassifier
|
||
:return:
|
||
"""
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
|
||
estimator = RandomForestClassifier(n_estimators=100)
|
||
classification_process(estimator, train_x, train_y_classification,
|
||
test_x, test_y_classification)
|
||
|
||
|
||
def sample_1032_4(show=True):
|
||
"""
|
||
10.3.2_4 猪老三使用分类预测股票涨跌：train_test_split
|
||
:return:
|
||
"""
|
||
from sklearn import metrics
|
||
from abupy import train_test_split
|
||
|
||
# noinspection PyShadowingNames
|
||
def train_test_split_xy(estimator, x, y, test_size=0.5,
|
||
random_state=0):
|
||
# 通过train_test_split将原始训练集随机切割为新训练集与测试集
|
||
train_x, test_x, train_y, test_y = \
|
||
train_test_split(x, y, test_size=test_size,
|
||
random_state=random_state)
|
||
|
||
if show:
|
||
print(x.shape, y.shape)
|
||
print(train_x.shape, train_y.shape)
|
||
print(test_x.shape, test_y.shape)
|
||
|
||
clf = estimator.fit(train_x, train_y)
|
||
predictions = clf.predict(test_x)
|
||
|
||
if show:
|
||
# 度量准确率
|
||
print("accuracy = %.2f" %
|
||
(metrics.accuracy_score(test_y, predictions)))
|
||
|
||
# 度量查准率
|
||
print("precision_score = %.2f" %
|
||
(metrics.precision_score(test_y, predictions)))
|
||
|
||
# 度量回收率
|
||
print("recall_score = %.2f" %
|
||
(metrics.recall_score(test_y, predictions)))
|
||
|
||
return test_y, predictions
|
||
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
estimator = RandomForestClassifier(n_estimators=100)
|
||
|
||
test_y, predictions = train_test_split_xy(estimator, train_x, train_y_classification)
|
||
return estimator, train_x, train_y_classification, test_y, predictions
|
||
|
||
|
||
def sample_1032_5():
|
||
"""
|
||
10.3.2_5 猪老三使用分类预测股票涨跌：混淆矩阵和roc曲线
|
||
:return:
|
||
"""
|
||
|
||
from sklearn import metrics
|
||
|
||
# noinspection PyShadowingNames
|
||
def confusion_matrix_with_report(test_y, predictions):
|
||
confusion_matrix = metrics.confusion_matrix(test_y, predictions)
|
||
# print("Confusion Matrix ", confusion_matrix)
|
||
print(" Predicted")
|
||
print(" | 0 | 1 |")
|
||
print(" |-----|-----|")
|
||
print(" 0 | %3d | %3d |" % (confusion_matrix[0, 0],
|
||
confusion_matrix[0, 1]))
|
||
print("Actual |-----|-----|")
|
||
print(" 1 | %3d | %3d |" % (confusion_matrix[1, 0],
|
||
confusion_matrix[1, 1]))
|
||
print(" |-----|-----|")
|
||
|
||
print(metrics.classification_report(test_y, predictions))
|
||
|
||
estimator, train_x, train_y_classification, test_y, predictions = sample_1032_4(show=False)
|
||
confusion_matrix_with_report(test_y, predictions)
|
||
from abupy import ABuMLExecute
|
||
ABuMLExecute.plot_roc_estimator(estimator, train_x, train_y_classification)
|
||
|
||
|
||
def sample_1033_1():
|
||
"""
|
||
10.3.3 通过决策树分类，绘制出决策图
|
||
这里需要安装dot graphviz，才能通过os.system("dot -T png graphviz.dot -o graphviz.png")生成png
|
||
:return:
|
||
"""
|
||
from sklearn.tree import DecisionTreeClassifier
|
||
from sklearn import tree
|
||
import os
|
||
|
||
estimator = DecisionTreeClassifier(max_depth=2, random_state=1)
|
||
|
||
# noinspection PyShadowingNames
|
||
def graphviz_tree(estimator, features, x, y):
|
||
if not hasattr(estimator, 'tree_'):
|
||
print('only tree can graphviz!')
|
||
return
|
||
|
||
estimator.fit(x, y)
|
||
# 将决策模型导出graphviz.dot文件
|
||
tree.export_graphviz(estimator.tree_, out_file='graphviz.dot',
|
||
feature_names=features)
|
||
# 通过dot将模型绘制决策图，保存png
|
||
os.system("dot -T png graphviz.dot -o graphviz.png")
|
||
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
# 这里会使用到特征的名称列pig_three_feature.columns[1:]
|
||
graphviz_tree(estimator, pig_three_feature.columns[1:], train_x,
|
||
train_y_classification)
|
||
|
||
import PIL.Image
|
||
PIL.Image.open('graphviz.png').show()
|
||
|
||
|
||
def sample_1033_2():
|
||
"""
|
||
10.3.3 特征的重要性排序及支持度评级
|
||
:return:
|
||
"""
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
# noinspection PyShadowingNames
|
||
def importances_coef_pd(estimator):
|
||
"""
|
||
特征的重要性
|
||
"""
|
||
if hasattr(estimator, 'feature_importances_'):
|
||
# 有feature_importances_的通过sort_values排序
|
||
return pd.DataFrame(
|
||
{'feature': list(pig_three_feature.columns[1:]),
|
||
'importance': estimator.feature_importances_}).sort_values('importance')
|
||
|
||
elif hasattr(estimator, 'coef_'):
|
||
# 有coef_的通过coef排序
|
||
return pd.DataFrame(
|
||
{"columns": list(pig_three_feature.columns)[1:], "coef": list(estimator.coef_.T)}).sort_values('coef')
|
||
else:
|
||
print('estimator not hasattr feature_importances_ or coef_!')
|
||
|
||
# 使用随机森林分类器
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
estimator = RandomForestClassifier(n_estimators=100)
|
||
# 训练数据模型
|
||
estimator.fit(train_x, train_y_classification)
|
||
# 对训练后的模型特征的重要度进行判定，重要程度由小到大，表10-4所示
|
||
print('importances_coef_pd(estimator):\n', importances_coef_pd(estimator))
|
||
|
||
from sklearn.feature_selection import RFE
|
||
|
||
# noinspection PyShadowingNames
|
||
def feature_selection(estimator, x, y):
|
||
"""
|
||
支持度评级
|
||
"""
|
||
selector = RFE(estimator)
|
||
selector.fit(x, y)
|
||
print('RFE selection')
|
||
print(pd.DataFrame(
|
||
{'support': selector.support_, 'ranking': selector.ranking_},
|
||
index=pig_three_feature.columns[1:]))
|
||
|
||
print('feature_selection(estimator, train_x, train_y_classification):\n',
|
||
feature_selection(estimator, train_x, train_y_classification))
|
||
|
||
|
||
"""
|
||
10.4 无监督机器学习
|
||
"""
|
||
|
||
|
||
def sample_1041():
|
||
"""
|
||
10.4.1 使用降维可视化数据
|
||
:return:
|
||
"""
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
from sklearn.decomposition import PCA
|
||
from abupy import ABuMLExecute
|
||
|
||
# noinspection PyShadowingNames
|
||
def plot_decision_function(estimator, x, y):
|
||
# pca进行降维，只保留2个特征序列
|
||
pca_2n = PCA(n_components=2)
|
||
x = pca_2n.fit_transform(x)
|
||
|
||
# 进行训练
|
||
estimator.fit(x, y)
|
||
plt.scatter(x[:, 0], x[:, 1], c=y, s=50, cmap='spring')
|
||
ABuMLExecute.plot_decision_boundary(
|
||
lambda p_x: estimator.predict(p_x), x, y)
|
||
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
estimator = RandomForestClassifier(n_estimators=100)
|
||
plot_decision_function(estimator, train_x, train_y_classification)
|
||
|
||
|
||
# noinspection PyTypeChecker
|
||
def sample_1042():
|
||
"""
|
||
10.4.2 猪老三使用聚类算法提高正确率
|
||
:return:
|
||
"""
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
# 使用随机森林作为分类器
|
||
from sklearn.ensemble import RandomForestClassifier
|
||
estimator = RandomForestClassifier(n_estimators=100)
|
||
estimator.fit(train_x, train_y_classification)
|
||
test_y_prdict_classification = estimator.predict(test_x)
|
||
|
||
from sklearn import metrics
|
||
print("accuracy = %.2f" % (
|
||
metrics.accuracy_score(test_y_classification,
|
||
test_y_prdict_classification)))
|
||
|
||
# 测试集feature即usFB的kl feature
|
||
pig_three_kmean_feature = kl_another_word_feature_test
|
||
# 测试集真实的涨跌结果test_y_classification
|
||
pig_three_kmean_feature['y'] = test_y_classification
|
||
# 使用刚刚的随机森林作为分类器的预测涨跌结果test_y_prdict_classification
|
||
pig_three_kmean_feature['y_prdict'] = test_y_prdict_classification
|
||
# 即生成一列新数据记录预测是否正确
|
||
pig_three_kmean_feature['y_same'] = np.where(
|
||
pig_three_kmean_feature['y'] ==
|
||
pig_three_kmean_feature['y_prdict'], 1, 0)
|
||
# 将feature中只保留刚刚得到的y_same
|
||
pig_three_kmean_feature = pig_three_kmean_feature.filter(['y_same'])
|
||
|
||
from sklearn.cluster import KMeans
|
||
|
||
# 使用刚刚得到的只有y_same列的数据赋值x_kmean
|
||
x_kmean = pig_three_kmean_feature.values
|
||
# n_clusters=2, 即只聚两类数据
|
||
kmean = KMeans(n_clusters=2)
|
||
kmean.fit(x_kmean)
|
||
# 将聚类标签赋予新的一列cluster
|
||
pig_three_kmean_feature['cluster'] = kmean.predict(x_kmean)
|
||
# 将周几这个特征合并过来
|
||
pig_three_kmean_feature['feature_date_week'] = \
|
||
kl_another_word_feature_test['feature_date_week']
|
||
# 表10-5所示
|
||
print('pig_three_kmean_feature.tail():\n', pig_three_kmean_feature.tail())
|
||
|
||
# 表10-6所示
|
||
print('pd.crosstab(pig_three_kmean_feature.feature_date_week, pig_three_kmean_feature.cluster):\n',
|
||
pd.crosstab(pig_three_kmean_feature.feature_date_week, pig_three_kmean_feature.cluster))
|
||
|
||
|
||
"""
|
||
10.5 梦醒时分
|
||
"""
|
||
|
||
|
||
def sample_105_0():
|
||
"""
|
||
10.5 AbuML
|
||
:return:
|
||
"""
|
||
global g_with_date_week_noise
|
||
g_with_date_week_noise = True
|
||
train_x, train_y_regress, train_y_classification, pig_three_feature, \
|
||
test_x, test_y_regress, test_y_classification, kl_another_word_feature_test = sample_1031_1()
|
||
|
||
from abupy import AbuML
|
||
# 通过x, y矩阵和特征的DataFrame对象组成AbuML
|
||
ml = AbuML(train_x, train_y_classification, pig_three_feature)
|
||
# 使用随机森林作为分类器
|
||
_ = ml.estimator.random_forest_classifier()
|
||
|
||
# 交织验证结果的正确率
|
||
print('ml.cross_val_accuracy_score():\n', ml.cross_val_accuracy_score())
|
||
# 特征的选择
|
||
print('ml.feature_selection():\n', ml.feature_selection())
|
||
|
||
|
||
"""
|
||
如下内容不能使用沙盒环境, 建议对照阅读：
|
||
abu量化文档－第十九节 数据源
|
||
第20节 美股交易UMP决策
|
||
"""
|
||
|
||
|
||
def sample_1051_0():
|
||
"""
|
||
10.5.1 回测中生成特征，切分训练测试集，成交买单快照: 数据准备
|
||
|
||
如果没有运行过abu量化文档－第十九节 数据源：中使用腾讯数据源进行数据更新，需要运行
|
||
如果运行过就不要重复运行了：
|
||
"""
|
||
from abupy import EMarketTargetType, EMarketSourceType, EDataCacheType
|
||
# 关闭沙盒数据环境
|
||
abupy.env.disable_example_env_ipython()
|
||
abupy.env.g_market_source = EMarketSourceType.E_MARKET_SOURCE_tx
|
||
abupy.env.g_data_cache_type = EDataCacheType.E_DATA_CACHE_CSV
|
||
# 首选这里预下载市场中所有股票的6年数据(做5年回测，需要预先下载6年数据)
|
||
abu.run_kl_update(start='2011-08-08', end='2017-08-08', market=EMarketTargetType.E_MARKET_TARGET_US)
|
||
|
||
|
||
def sample_1051_1(from_cache=False, show=True):
|
||
"""
|
||
10.5.1 回测中生成特征，切分训练测试集，成交买单快照: 数据准备
|
||
:return:
|
||
"""
|
||
from abupy import AbuMetricsBase
|
||
from abupy import AbuFactorBuyBreak
|
||
from abupy import AbuFactorAtrNStop
|
||
from abupy import AbuFactorPreAtrNStop
|
||
from abupy import AbuFactorCloseAtrNStop
|
||
|
||
# 关闭沙盒数据环境
|
||
abupy.env.disable_example_env_ipython()
|
||
from abupy import EMarketDataFetchMode
|
||
# 因为sample_94_1下载了预先数据，使用缓存，设置E_DATA_FETCH_FORCE_LOCAL，实际上run_kl_update最后会把设置set到FORCE_LOCAL
|
||
abupy.env.g_data_fetch_mode = EMarketDataFetchMode.E_DATA_FETCH_FORCE_LOCAL
|
||
|
||
# 设置选股因子，None为不使用选股因子
|
||
stock_pickers = None
|
||
# 买入因子依然延用向上突破因子
|
||
buy_factors = [{'xd': 60, 'class': AbuFactorBuyBreak},
|
||
{'xd': 42, 'class': AbuFactorBuyBreak}]
|
||
|
||
# 卖出因子继续使用上一章使用的因子
|
||
sell_factors = [
|
||
{'stop_loss_n': 1.0, 'stop_win_n': 3.0,
|
||
'class': AbuFactorAtrNStop},
|
||
{'class': AbuFactorPreAtrNStop, 'pre_atr_n': 1.5},
|
||
{'class': AbuFactorCloseAtrNStop, 'close_atr_n': 1.5}
|
||
]
|
||
|
||
# 回测生成买入时刻特征
|
||
abupy.env.g_enable_ml_feature = True
|
||
# 回测将symbols切割分为训练集数据和测试集数据
|
||
abupy.env.g_enable_train_test_split = True
|
||
# 下面设置回测时切割训练集，测试集使用的切割比例参数，默认为10，即切割为10份，9份做为训练，1份做为测试，
|
||
# 由于美股股票数量多，所以切割分为4份，3份做为训练集，1份做为测试集
|
||
abupy.env.g_split_tt_n_folds = 4
|
||
|
||
from abupy import EStoreAbu
|
||
if from_cache:
|
||
abu_result_tuple = \
|
||
abu.load_abu_result_tuple(n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
|
||
custom_name='train_us')
|
||
else:
|
||
# 初始化资金500万，资金管理依然使用默认atr
|
||
read_cash = 5000000
|
||
# 每笔交易的买入基数资金设置为万分之15
|
||
abupy.beta.atr.g_atr_pos_base = 0.0015
|
||
# 使用run_loop_back运行策略，因子使用和之前一样，
|
||
# choice_symbols=None为全市场回测，5年历史数据回测
|
||
abu_result_tuple, _ = abu.run_loop_back(read_cash,
|
||
buy_factors, sell_factors,
|
||
stock_pickers,
|
||
choice_symbols=None,
|
||
start='2012-08-08', end='2017-08-08')
|
||
# 把运行的结果保存在本地，以便之后分析回测使用，保存回测结果数据代码如下所示
|
||
abu.store_abu_result_tuple(abu_result_tuple, n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
|
||
custom_name='train_us')
|
||
|
||
if show:
|
||
metrics = AbuMetricsBase(*abu_result_tuple)
|
||
metrics.fit_metrics()
|
||
metrics.plot_returns_cmp(only_show_returns=True)
|
||
|
||
"＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊＊"
|
||
abupy.env.g_enable_train_test_split = False
|
||
# 使用切割好的测试数据
|
||
abupy.env.g_enable_last_split_test = True
|
||
|
||
from abupy import EStoreAbu
|
||
if from_cache:
|
||
abu_result_tuple_test = \
|
||
abu.load_abu_result_tuple(n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
|
||
custom_name='test_us')
|
||
else:
|
||
read_cash = 5000000
|
||
abupy.beta.atr.g_atr_pos_base = 0.007
|
||
choice_symbols = None
|
||
abu_result_tuple_test, kl_pd_manager_test = abu.run_loop_back(read_cash,
|
||
buy_factors, sell_factors, stock_pickers,
|
||
choice_symbols=choice_symbols, start='2012-08-08',
|
||
end='2017-08-08')
|
||
abu.store_abu_result_tuple(abu_result_tuple_test, n_folds=5, store_type=EStoreAbu.E_STORE_CUSTOM_NAME,
|
||
custom_name='test_us')
|
||
|
||
if show:
|
||
metrics = AbuMetricsBase(*abu_result_tuple_test)
|
||
metrics.fit_metrics()
|
||
metrics.plot_returns_cmp(only_show_returns=True)
|
||
print(abu_result_tuple.orders_pd[abu_result_tuple.orders_pd.result != 0].head())
|
||
|
||
return abu_result_tuple, abu_result_tuple_test
|
||
|
||
|
||
# noinspection PyUnresolvedReferences
|
||
def sample_1052():
|
||
"""
|
||
10.5.2 基于特征的交易预测
|
||
:return:
|
||
"""
|
||
# 需要在有缓存的情况下运行
|
||
abu_result_tuple, _ = sample_1051_1(from_cache=True, show=False)
|
||
|
||
from abupy.UmpBu.ABuUmpMainMul import UmpMulFiter
|
||
mul = UmpMulFiter(orders_pd=abu_result_tuple.orders_pd, scaler=False)
|
||
print('mul.df.head():\n', mul.df.head())
|
||
|
||
# 默认使用svm作为分类器
|
||
print('decision_tree_classifier cv please wait...')
|
||
mul.estimator.decision_tree_classifier()
|
||
mul.cross_val_accuracy_score()
|
||
|
||
# 默认使用svm作为分类器
|
||
print('knn_classifier cv please wait...')
|
||
# 默认使用svm作为分类器, 改分类器knn
|
||
mul.estimator.knn_classifier()
|
||
mul.cross_val_accuracy_score()
|
||
|
||
from abupy.UmpBu.ABuUmpMainBase import UmpDegFiter
|
||
deg = UmpDegFiter(orders_pd=abu_result_tuple.orders_pd)
|
||
print('deg.df.head():\n', deg.df.head())
|
||
|
||
print('xgb_classifier cv please wait...')
|
||
# 分类器使用GradientBoosting
|
||
deg.estimator.xgb_classifier()
|
||
deg.cross_val_accuracy_score()
|
||
|
||
print('adaboost_classifier cv please wait...')
|
||
# 分类器使用adaboost
|
||
deg.estimator.adaboost_classifier(base_estimator=None)
|
||
deg.cross_val_accuracy_score()
|
||
|
||
print('train_test_split_xy please wait...')
|
||
deg.train_test_split_xy()
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sample_102()
|
||
# sample_103_0()
|
||
# sample_1031_1()
|
||
# sample_1031_2()
|
||
# sample_1031_3()
|
||
# sample_1031_4()
|
||
# sample_1032_1()
|
||
# sample_1032_2()
|
||
# sample_1032_3()
|
||
# sample_1032_4()
|
||
# sample_1032_5()
|
||
# sample_1033_1()
|
||
# sample_1033_2()
|
||
# sample_1041()
|
||
# sample_1042()
|
||
# sample_105_0()
|
||
# sample_1051_0()
|
||
# sample_1051_1(from_cache=True)
|
||
# sample_1051_1(from_cache=False)
|
||
# sample_1052()
|