{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport lightgbm as lgb\nimport gresearch_crypto\nimport datetime\nfrom sklearn.preprocessing import RobustScaler\nfrom sklearn.metrics import mean_absolute_error as MSE\nimport warnings\nwarnings.filterwarnings('ignore')\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\nSEED = 2021\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-15T03:34:41.574422Z","iopub.execute_input":"2022-01-15T03:34:41.575331Z","iopub.status.idle":"2022-01-15T03:34:41.582495Z","shell.execute_reply.started":"2022-01-15T03:34:41.575288Z","shell.execute_reply":"2022-01-15T03:34:41.581844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fix_all_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nfix_all_seeds(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T03:34:41.584377Z","iopub.execute_input":"2022-01-15T03:34:41.584992Z","iopub.status.idle":"2022-01-15T03:34:41.592946Z","shell.execute_reply.started":"2022-01-15T03:34:41.584937Z","shell.execute_reply":"2022-01-15T03:34:41.592283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n#asset_weight_dict =df_asset_details[['Asset_ID','Weight']].to_dict()","metadata":{"execution":{"iopub.status.busy":"2022-01-15T03:34:41.594640Z","iopub.execute_input":"2022-01-15T03:34:41.595223Z","iopub.status.idle":"2022-01-15T03:34:41.602232Z","shell.execute_reply.started":"2022-01-15T03:34:41.595179Z","shell.execute_reply":"2022-01-15T03:34:41.601516Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def log_return(series, periods=5):\n    return np.log(series).diff(periods=periods)\n\ndef get_features(df):\n    \n    times = pd.to_datetime(df.index,unit=\"s\",infer_datetime_format=True)\n    df[\"hour\"] = times.hour  \n    df[\"dayofweek\"] = times.dayofweek \n    df[\"day\"] = times.day \n    for i in range(14):\n        df[f'Upper_Shadow_{i}'] =  df[f'High_{i}'] - np.maximum(df[f'Close_{i}'], df[f'Open_{i}'])\n        df[f'hlco_ratio_{i}'] =  (df[f'High_{i}'] - df[f'Low_{i}'])/(df[f'Close_{i}']-df[f'Open_{i}'])\n        df[f'Lower_Shadow_{i}'] =  np.minimum(df[f'Close_{i}'], df[f'Open_{i}']) - df[f'Low_{i}']\n        df[f'diff_upper{i}'] =df[f'Upper_Shadow_{i}'].diff().fillna(0)\n        df[f'diff_ratio_{i}'] =df[f'hlco_ratio_{i}'].diff().fillna(0)\n        df[f'diff_Shadow_{i}'] = df[f'Lower_Shadow_{i}'].diff().fillna(0)\n        \n        df[f'open2close_{i}'] = df[f'Close_{i}'] / df[f'Open_{i}']\n        df[f'high2low_{i}'] = df[f'High_{i}'] / df[f'Low_{i}']\n    return df ","metadata":{"execution":{"iopub.status.busy":"2022-01-15T03:34:41.604483Z","iopub.execute_input":"2022-01-15T03:34:41.604855Z","iopub.status.idle":"2022-01-15T03:34:41.614562Z","shell.execute_reply.started":"2022-01-15T03:34:41.604820Z","shell.execute_reply":"2022-01-15T03:34:41.613860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pickle\nwith open('../input/greserchmodel/models','rb') as web:\n    models = pickle.load(web)","metadata":{"execution":{"iopub.status.busy":"2022-01-15T03:35:19.195520Z","iopub.execute_input":"2022-01-15T03:35:19.195801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print( 'start',datetime.datetime.now().strftime('%Y年%m月%d日 %H:%M:%S'))\n\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\nfor i, (df_test, df_pred) in enumerate(iter_test):\n    df_test = df_test.fillna(-1)\n    df_test.VWAP = df_test.VWAP.replace(np.inf,0)\n    df_test.VWAP = df_test.VWAP.replace(-np.inf,0)\n    \n    dfs = df_test.iloc[:, 2:-1].apply(lambda x: (x-x.mean())/x.std(), axis=0)\n    df_test.iloc[:, 2:-1] = dfs \n\n    #df_test['Weight'] = df_test['Asset_ID'].map(asset_weight_dict['Weight']) /10\n    test = pd.DataFrame(index=(df_test['timestamp'].unique()))\n    test['timestamp'] = df_test['timestamp'].unique()\n    for i in range(14):\n        data = df_test[df_test.Asset_ID == i].copy()\n        for x in data.columns:\n            if x !='timestamp':\n                if x != 'Asset_ID' and  x!='group_num' and x!='row_id':\n                    test[f\"{x}_{i}\"]  = data[f\"{x}\"] \n            else:\n                data.set_index(\"timestamp\",inplace=True)  \n\n    #Asset_ID = df_test.Asset_ID.values\n    df_proc = get_features(test)\n    \n    #df_valid.set_index(\"timestamp\",inplace=True) \n    #df_proc = df_proc.fillna(-1)\n    #x_pred = pd.DataFrame()\n    timestamp= pd.DataFrame()\n    for x in df_test.Asset_ID.unique():\n        #print('Asset_Id=',x)\n        model = models[x]\n        \n        data = df_test[df_test.Asset_ID == x]\n        #timestamp = data['timestamp']\n        #row_id = data['row_id'].values\n        \n        x_test = df_proc[df_proc.index.isin(data.timestamp.values)]\n        \n        #print('Asset_Id=',x,'  predict')\n        x_test['pred'] = model.predict(x_test)\n        \n        x_test['row_id'] =data.row_id.values\n        #x_pred = pd.concat([x_pred,x_test])\n      \n        for j,row in x_test.iterrows():\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = row.pred\n        \n    #if x == 0:\n    #    display(df_pred)\n        \n    env.predict(df_pred)\n    \nprint( 'end',datetime.datetime.now().strftime('%Y年%m月%d日 %H:%M:%S'))","metadata":{"execution":{"iopub.status.busy":"2022-01-15T03:34:41.643044Z","iopub.status.idle":"2022-01-15T03:34:41.643546Z","shell.execute_reply.started":"2022-01-15T03:34:41.643304Z","shell.execute_reply":"2022-01-15T03:34:41.643328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_pred","metadata":{"execution":{"iopub.status.busy":"2022-01-15T03:34:41.644770Z","iopub.status.idle":"2022-01-15T03:34:41.646453Z","shell.execute_reply.started":"2022-01-15T03:34:41.646276Z","shell.execute_reply":"2022-01-15T03:34:41.646296Z"},"trusted":true},"execution_count":null,"outputs":[]}]}