{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+En8*ter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-20T05:17:03.199789Z","iopub.status.idle":"2022-01-20T05:17:03.200521Z","shell.execute_reply.started":"2022-01-20T05:17:03.200079Z","shell.execute_reply":"2022-01-20T05:17:03.200121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nfrom datetime import datetime\nimport time\nimport os \n\nimport pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:17:03.280775Z","iopub.execute_input":"2022-01-20T05:17:03.281194Z","iopub.status.idle":"2022-01-20T05:17:05.403565Z","shell.execute_reply.started":"2022-01-20T05:17:03.281150Z","shell.execute_reply":"2022-01-20T05:17:05.402605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:17:05.405220Z","iopub.execute_input":"2022-01-20T05:17:05.405434Z","iopub.status.idle":"2022-01-20T05:17:05.411405Z","shell.execute_reply.started":"2022-01-20T05:17:05.405407Z","shell.execute_reply":"2022-01-20T05:17:05.410431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# load data & data preprocessing","metadata":{}},{"cell_type":"code","source":"data_folder = \"/kaggle/input/g-research-crypto-forecasting/\"\n\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\nstart_time = totimestamp('01/01/2021')\nend_time = totimestamp('21/09/2021')\ntrain_index = slice(start_time,totimestamp('01/07/2021')-60)\ntest_index = slice(totimestamp('01/07/2021'),end_time)\n\nall_train = pd.read_csv(data_folder+'train.csv').set_index(\"timestamp\").loc[start_time : end_time]\n\nasset_details = pd.read_csv(data_folder+'asset_details.csv')\nasset_details['w'] = asset_details['Weight']/asset_details['Weight'].sum()\nasset_details = asset_details.set_index('Asset_ID').sort_index()","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:17:05.412529Z","iopub.execute_input":"2022-01-20T05:17:05.413149Z","iopub.status.idle":"2022-01-20T05:18:08.642285Z","shell.execute_reply.started":"2022-01-20T05:17:05.413027Z","shell.execute_reply":"2022-01-20T05:18:08.641418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"all_train","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:08.643364Z","iopub.execute_input":"2022-01-20T05:18:08.643588Z","iopub.status.idle":"2022-01-20T05:18:08.668073Z","shell.execute_reply.started":"2022-01-20T05:18:08.643560Z","shell.execute_reply":"2022-01-20T05:18:08.667552Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.pivot_table(all_train[['Asset_ID','Open']],values = 'Open',index =all_train.index ,columns = ['Asset_ID'])\n","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:08.670283Z","iopub.execute_input":"2022-01-20T05:18:08.670625Z","iopub.status.idle":"2022-01-20T05:18:08.673637Z","shell.execute_reply.started":"2022-01-20T05:18:08.670597Z","shell.execute_reply":"2022-01-20T05:18:08.673072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# del t1","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:08.689727Z","iopub.execute_input":"2022-01-20T05:18:08.690329Z","iopub.status.idle":"2022-01-20T05:18:08.693729Z","shell.execute_reply.started":"2022-01-20T05:18:08.690294Z","shell.execute_reply":"2022-01-20T05:18:08.693034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = {}\nfor col in all_train.columns[1:]:\n#     features[col] = pd.pivot_table(all_train[['timestamp','Asset_ID',col]],values = col,index ='timestamp' ,columns = ['Asset_ID'])\n    features[col] = pd.pivot_table(all_train[['Asset_ID',col]],values = col,index =all_train.index ,columns = ['Asset_ID'])\n    features[col] = features[col].reindex(range(start_time,end_time+60,60),method='pad')\n    print(f'finish {col}')\n    \nidx = features['Open'].index\ncols = features['Open'].columns","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:08.696638Z","iopub.execute_input":"2022-01-20T05:18:08.696916Z","iopub.status.idle":"2022-01-20T05:18:35.973410Z","shell.execute_reply.started":"2022-01-20T05:18:08.696881Z","shell.execute_reply":"2022-01-20T05:18:35.972326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# source\nprint('source',features.keys())\n\n# 缺失值比率\nprint('缺失值比率')\nfor k,v in features.items():\n    print(k,(v.isnull().sum() / v.shape[0]).mean()  )\n","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:35.974795Z","iopub.execute_input":"2022-01-20T05:18:35.975106Z","iopub.status.idle":"2022-01-20T05:18:36.083175Z","shell.execute_reply.started":"2022-01-20T05:18:35.975057Z","shell.execute_reply":"2022-01-20T05:18:36.082286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features['Open'].head()","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:36.084423Z","iopub.execute_input":"2022-01-20T05:18:36.084643Z","iopub.status.idle":"2022-01-20T05:18:36.105427Z","shell.execute_reply.started":"2022-01-20T05:18:36.084615Z","shell.execute_reply":"2022-01-20T05:18:36.104638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"缺失值主要因为很多小币在早期没有数据","metadata":{}},{"cell_type":"code","source":"# def ts_corr(df):\n#     return  np.average(df.corrwith(features['Target']),weights = asset_details['w'] ) \n\ndef corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))\n\ndef log_return(series, periods):\n    return np.log(series).diff(periods=periods)\n\ndef get_mkt_and_residual_ret(ret):\n    market_return = pd.Series(np.nanmean(ret.mul(asset_details['w'],axis=1) ,axis=1),index = idx)\n    a = ret.mul(market_return,axis=0).rolling(3750,min_periods=3750).mean()\n    b = (market_return*market_return).rolling(3750,min_periods=3750).mean()\n    beta = a.div(b,axis=0)\n    re_target = ret - beta.mul(market_return,axis=0)\n    return market_return,re_target\n\nupper_shadow = (features['High'] - np.maximum(features['Close'],features['Open']))/features['Close']\nlower_shadow = (np.minimum(features['Close'],features['Open']) - features['Low'])/features['Close']\n\nlogret15 = log_return(features['Close'],15)\nlogret15_mkt,logret15_re = get_mkt_and_residual_ret(log_return(features['Close'],15))\nlogret15_mkt= pd.DataFrame(np.tile(logret15_mkt.values,(14,1)).T,index=idx, columns = cols)\n\nlogret60 = log_return(features['Close'],60)\nlogret60_mkt,logret60_re = get_mkt_and_residual_ret(log_return(features['Close'],60))\nlogret60_mkt= pd.DataFrame(np.tile(logret60_mkt.values,(14,1)).T,index=idx, columns = cols)\n\nlogret100 = log_return(features['Close'],100)\nlogret100_mkt,logret100_re = get_mkt_and_residual_ret(log_return(features['Close'],100))\nlogret100_mkt= pd.DataFrame(np.tile(logret100_mkt.values,(14,1)).T,index=idx, columns = cols)","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:36.107406Z","iopub.execute_input":"2022-01-20T05:18:36.108235Z","iopub.status.idle":"2022-01-20T05:18:37.571843Z","shell.execute_reply.started":"2022-01-20T05:18:36.108164Z","shell.execute_reply":"2022-01-20T05:18:37.571041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# feats = [(features['High'] - np.maximum(features['Close'],features['Open'])),\n#         (np.minimum(features['Close'],features['Open']) - features['Low']),\n#          logret15] #改进前的因子\n\nfeats = [upper_shadow,lower_shadow,\n         logret15_mkt, logret60_mkt, logret100_mkt,\n         logret15_re, logret60_re, logret100_re,\n        logret15, logret60, logret100]\n\nfeats = [feat.unstack() for feat in feats]\nX = pd.concat(feats,axis=1)\ny = features['Target'].unstack()\n\n\nX_train = X.loc[(slice(0,13),train_index),:]\ny_train = y.loc[slice(0,13),train_index]\nX_test = X.loc[(slice(0,13),test_index),:]\ny_test = y.loc[slice(0,13),test_index]\n\n# def split(X,y,train_ratio):\n#     train_num = int(X.shape[0]*train_ratio)\n#     return X.iloc[:train_num],X.iloc[train_num:], y.iloc[:train_num],y.iloc[train_num:]\n\n# X_train, X_valid, y_train, y_valid = split(X_train,y_train,0.8)","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:37.572999Z","iopub.execute_input":"2022-01-20T05:18:37.573325Z","iopub.status.idle":"2022-01-20T05:18:48.250307Z","shell.execute_reply.started":"2022-01-20T05:18:37.573295Z","shell.execute_reply":"2022-01-20T05:18:48.249339Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape[0]+X_test.shape[0] == X.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:48.251623Z","iopub.execute_input":"2022-01-20T05:18:48.251927Z","iopub.status.idle":"2022-01-20T05:18:48.258682Z","shell.execute_reply.started":"2022-01-20T05:18:48.251886Z","shell.execute_reply":"2022-01-20T05:18:48.257819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_train.shape[0]+y_test.shape[0] == y.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:48.259847Z","iopub.execute_input":"2022-01-20T05:18:48.260109Z","iopub.status.idle":"2022-01-20T05:18:48.273514Z","shell.execute_reply.started":"2022-01-20T05:18:48.260081Z","shell.execute_reply":"2022-01-20T05:18:48.272652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X.shape[0] == y.shape[0]","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:48.275811Z","iopub.execute_input":"2022-01-20T05:18:48.276418Z","iopub.status.idle":"2022-01-20T05:18:48.287637Z","shell.execute_reply.started":"2022-01-20T05:18:48.276371Z","shell.execute_reply":"2022-01-20T05:18:48.286554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# model","metadata":{}},{"cell_type":"markdown","source":"## weighter correlation","metadata":{}},{"cell_type":"code","source":"def corr(a, b, w):\n    cov = lambda x, y: np.sum(w * (x - np.average(x, weights=w)) * (y - np.average(y, weights=w))) / np.sum(w)\n    return cov(a, b) / np.sqrt(cov(a, a) * cov(b, b))","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:48.289117Z","iopub.execute_input":"2022-01-20T05:18:48.289650Z","iopub.status.idle":"2022-01-20T05:18:48.298388Z","shell.execute_reply.started":"2022-01-20T05:18:48.289596Z","shell.execute_reply":"2022-01-20T05:18:48.297376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w = pd.Series(index = X_test.index)\nfor i in asset_details['Weight'].index:\n    w.loc[i] = asset_details['Weight'][i]\nw","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:48.299619Z","iopub.execute_input":"2022-01-20T05:18:48.299889Z","iopub.status.idle":"2022-01-20T05:18:48.378443Z","shell.execute_reply.started":"2022-01-20T05:18:48.299857Z","shell.execute_reply":"2022-01-20T05:18:48.377877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## big model","metadata":{}},{"cell_type":"markdown","source":"### linear ","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\n# implement basic ML baseline (one per asset)\nlr = LinearRegression()\nlr.fit(X_train.ffill().fillna(0),y_train.ffill().fillna(0))\ny_lr_pred = lr.predict(X_test.ffill().fillna(0))\nprint(\"y_lr_pred : \",corr(np.nan_to_num(y_lr_pred),np.nan_to_num(y_test.values),np.nan_to_num(np.array(w.to_list()))) )","metadata":{"execution":{"iopub.status.busy":"2022-01-16T02:45:25.183481Z","iopub.execute_input":"2022-01-16T02:45:25.183686Z","iopub.status.idle":"2022-01-16T02:45:27.522855Z","shell.execute_reply.started":"2022-01-16T02:45:25.183659Z","shell.execute_reply":"2022-01-16T02:45:27.521811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### lightgbm","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb\n\nlgb_train = lgb.Dataset(X_train, y_train)\nlgb_eval = lgb.Dataset(X_test, y_test, reference=lgb_train)\n\n# specify your configurations as a dict\nparams = {\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': {'l2', 'l1'},\n    'num_leaves': 31,\n    'learning_rate': 0.2,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0\n}\n\nprint('Starting training...')\n# train\ngbm = lgb.train(params,\n                lgb_train,\n                num_boost_round=500,\n                valid_sets=[lgb_train,lgb_eval],\n                early_stopping_rounds=50,\n               verbose_eval=50)\n\ny_lgb_pred = gbm.predict(X_test, num_iteration=gbm.best_iteration)\nprint(\"y_lr_pred : \",corr(np.nan_to_num(y_lgb_pred),np.nan_to_num(y_test.values),np.nan_to_num(np.array(w.to_list()))) )","metadata":{"execution":{"iopub.status.busy":"2022-01-16T02:45:27.523841Z","iopub.execute_input":"2022-01-16T02:45:27.524031Z","iopub.status.idle":"2022-01-16T02:45:37.414683Z","shell.execute_reply.started":"2022-01-16T02:45:27.524007Z","shell.execute_reply":"2022-01-16T02:45:37.413687Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# small model","metadata":{}},{"cell_type":"code","source":"X_trains = {}\ny_trains = {}\nX_tests = {}\ny_tests = {}\nfor i in range(14):\n    X_trains[i] = X_train.loc[(i),:]\n    y_trains[i] = y_train.loc[(i),:]\n    X_tests[i] = X_test.loc[(i),:]\n    y_tests[i] = y_test.loc[(i),:]","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:18:48.379332Z","iopub.execute_input":"2022-01-20T05:18:48.380048Z","iopub.status.idle":"2022-01-20T05:18:49.635232Z","shell.execute_reply.started":"2022-01-20T05:18:48.380014Z","shell.execute_reply":"2022-01-20T05:18:49.634279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\n\nlr_models = {}\n\ndef get_lr_model(X_train,y_train):\n    lr = LinearRegression()\n    lr.fit(X_train.ffill().fillna(0),y_train.ffill().fillna(0))\n    return lr\n\nfor (i,X_train),(_,y_train) in zip(X_trains.items(),y_trains.items()):\n    lr_models[i] = get_lr_model(X_train,y_train)\n\ny_lr_preds = []\nfor i,lr_model in lr_models.items():\n    y_lr_preds.append(lr_model.predict(X_tests[i].ffill().fillna(0)))\n\nprint(\"small seperate model , y_lr_preds : \",corr(np.nan_to_num(np.array(y_lr_preds).flatten()),np.nan_to_num(y_test.values),np.nan_to_num(np.array(w.to_list()))) )","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:21:18.581846Z","iopub.execute_input":"2022-01-20T05:21:18.583085Z","iopub.status.idle":"2022-01-20T05:21:21.226138Z","shell.execute_reply.started":"2022-01-20T05:21:18.583040Z","shell.execute_reply":"2022-01-20T05:21:21.225282Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import lightgbm as lgb\nlgb_models = {}\nparams = {\n    'boosting_type': 'gbdt',\n    'objective': 'regression',\n    'metric': {'l2', 'l1'},\n    'num_leaves': 31,\n    'learning_rate': 0.2,\n    'feature_fraction': 0.9,\n    'bagging_fraction': 0.8,\n    'bagging_freq': 5,\n    'verbose': 0\n}\n\ndef get_lgb_model(X_train,y_train):\n    lgb_train = lgb.Dataset(X_train, y_train)\n    print('Starting training...')\n    # train\n    gbm = lgb.train(params,\n                    lgb_train,\n                    num_boost_round=500,\n                    valid_sets=[lgb_train],\n                    early_stopping_rounds=50,\n                   verbose_eval=50)\n    return gbm\n\nfor (i,X_train),(_,y_train) in zip(X_trains.items(),y_trains.items()):\n    lgb_models[i] = get_lgb_model(X_train,y_train)\n\ny_lgb_preds = []\nfor i,lgb_model in lgb_models.items():\n    y_lgb_preds.append(lgb_model.predict(X_tests[i], num_iteration=lgb_model.best_iteration))\n    \nprint(\"small seperate model , y_lgb_preds : \",corr(np.nan_to_num(np.array(y_lgb_preds).flatten()),np.nan_to_num(y_test.values),np.nan_to_num(np.array(w.to_list()))) )","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:21:25.335823Z","iopub.execute_input":"2022-01-20T05:21:25.336101Z","iopub.status.idle":"2022-01-20T05:22:38.419559Z","shell.execute_reply.started":"2022-01-20T05:21:25.336064Z","shell.execute_reply":"2022-01-20T05:22:38.418566Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"small seperate model , y_lgb_preds : \",corr(np.nan_to_num(np.array(y_lgb_preds).flatten()),np.nan_to_num(y_test.values),np.nan_to_num(np.array(w.to_list()))) )","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:22:38.429476Z","iopub.execute_input":"2022-01-20T05:22:38.434034Z","iopub.status.idle":"2022-01-20T05:22:38.904246Z","shell.execute_reply.started":"2022-01-20T05:22:38.433975Z","shell.execute_reply":"2022-01-20T05:22:38.903349Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"lgb single model for asset 0 \")\nnp.corrcoef(np.nan_to_num(y_lgb_preds[0]),np.nan_to_num(y_tests[0].values))","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:26:49.063300Z","iopub.execute_input":"2022-01-20T05:26:49.066428Z","iopub.status.idle":"2022-01-20T05:26:49.080038Z","shell.execute_reply.started":"2022-01-20T05:26:49.066371Z","shell.execute_reply":"2022-01-20T05:26:49.079306Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"linear single model for asset 0 \")\nnp.corrcoef(np.nan_to_num(y_lr_preds[0]),np.nan_to_num(y_tests[0].values))","metadata":{"execution":{"iopub.status.busy":"2022-01-20T05:26:47.313496Z","iopub.execute_input":"2022-01-20T05:26:47.314799Z","iopub.status.idle":"2022-01-20T05:26:47.328086Z","shell.execute_reply.started":"2022-01-20T05:26:47.314725Z","shell.execute_reply":"2022-01-20T05:26:47.327178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# submission","metadata":{}},{"cell_type":"code","source":"# import gresearch_crypto\n# import traceback\n\n# env = gresearch_crypto.make_env()\n# iter_test = env.iter_test()\n# # model = lr\n\n# for i, (df_test, df_pred) in enumerate(iter_test):\n#     for j , row in df_test.iterrows():        \n#         try:\n#             print(row)\n#             x_test = get_features(row)\n#             y_pred = model.predict(x_test.ffill.fillna(0))\n#             df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = y_pred\n#         except:\n#             df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n#             traceback.print_exc()\n    \n#     env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-16T02:45:37.417075Z","iopub.execute_input":"2022-01-16T02:45:37.417976Z","iopub.status.idle":"2022-01-16T02:45:37.555238Z","shell.execute_reply.started":"2022-01-16T02:45:37.417931Z","shell.execute_reply":"2022-01-16T02:45:37.55353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### recreating target","metadata":{}},{"cell_type":"code","source":"# # r 根据题目定义严格实现 Ra(t)=log(Pa(t+16) / Pa(t+1))\n# ret =  np.log(features['Close'].shift(-16)/features['Close'].shift(-1))\n# market_return = pd.Series(np.nanmean(ret.mul(asset_details['w'],axis=1)\n#                                      ,axis=1),index = idx)\n\n# # beta 根据题目定义实现 beta = m*r / m*m\n# a = ret.mul(market_return,axis=0).rolling(3750,min_periods=3750).mean()\n# b = (market_return*market_return).rolling(3750,min_periods=3750).mean()\n# beta = a.div(b,axis=0)\n\n# #\n# re_target = ret - beta.mul(market_return,axis=0)\n# print(\"mean mae\",(re_target - features['Target']).abs().mean())\n# print(\"max mae\",(re_target - features['Target']).abs().max())\n# print(\"min mae\",(re_target - features['Target']).abs().min())","metadata":{"execution":{"iopub.status.busy":"2022-01-16T02:45:37.55627Z","iopub.status.idle":"2022-01-16T02:45:37.556614Z","shell.execute_reply.started":"2022-01-16T02:45:37.556439Z","shell.execute_reply":"2022-01-16T02:45:37.556461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## lgbm tune","metadata":{}},{"cell_type":"code","source":"# from sklearn.model_selection import GridSearchCV\n\n# def get_model(X,y):\n#     model = LGBMRegressor()\n#     model.fit(X, y)\n#     return X, y, model\n\n# parameters = {\n# #     'max_depth': range (2, 10, 1),\n#     'num_leaves': range(21, 161, 10),\n#     'learning_rate': [0.1, 0.01, 0.05]\n# }\n\n# grid_search = GridSearchCV(\n#         estimator=get_model(X_btc_train_scaled,y_btc_train)[2], # bitcoin\n#         param_grid=parameters,\n#         n_jobs = -1,\n#         cv = 5,\n#         verbose=True\n#     )\n# grid_search.fit(X_btc_train_scaled, y_btc_train)\n# new_model = grid_search.best_estimator_\n# grid_search.best_estimator_\n\n# y_pred_lgbm_tune_btc = new_model.predict(X_btc_test_scaled)\n# print('Test score for LGBM tune model: BTC', f\"{np.corrcoef(y_pred_lgbm_tune_btc, y_btc_test)[0,1]:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2022-01-16T02:45:37.558411Z","iopub.status.idle":"2022-01-16T02:45:37.558749Z","shell.execute_reply.started":"2022-01-16T02:45:37.558578Z","shell.execute_reply":"2022-01-16T02:45:37.558599Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## simple nn","metadata":{}},{"cell_type":"code","source":"# import tensorflow as tf\n# from tensorflow.keras.models import Sequential\n# from tensorflow.keras.layers import LSTM\n# from tensorflow.keras.layers import Dense\n# from tensorflow.keras.layers import Dropout\n# from tensorflow.keras import Model\n# from tensorflow.keras import Input\n# tf.__version__\n\n# model1 = tf.keras.models.Sequential()\n# model1.add(tf.keras.Input(shape=(4,)))\n# model1.add(tf.keras.layers.Dense(100, activation='relu'))\n# model1.add(tf.keras.layers.Dense(100, activation='relu'))\n# model1.add(tf.keras.layers.Dense(100, activation='relu'))\n# model1.add(tf.keras.layers.Dense(32, activation='relu'))\n# model1.add(tf.keras.layers.Dense(16, activation='relu'))\n# model1.add(tf.keras.layers.Dense(1))\n# model1.output_shape\n# model1.compile(loss='mse',optimizer='adam')\n# print(model1.summary())\n# model1.fit(X_train, y_train.ffill(), batch_size=3000, epochs=5)\n\n# y_pred = model1.predict(X_valid)\n# np.corrcoef(y_pred.flatten(),y_valid.ffill())[0,1]","metadata":{"execution":{"iopub.status.busy":"2022-01-16T02:45:37.560275Z","iopub.status.idle":"2022-01-16T02:45:37.560594Z","shell.execute_reply.started":"2022-01-16T02:45:37.560426Z","shell.execute_reply":"2022-01-16T02:45:37.560446Z"},"trusted":true},"execution_count":null,"outputs":[]}]}