{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# [inference] 3rd place solution\n\nI will post a description of the model in discussion. Training is done in another notebook.\n* Training: https://www.kaggle.com/sugghi/training-3rd-place-solution/\n\nI referred to various notebooks when coding. In particular, the following notebook was used directly.\n* Submitting Lagged Features via API：\nhttps://www.kaggle.com/tomforbes/gresearch-submitting-lagged-features-via-api\n\nIn addition, local api published by @jagofc helped me a lot in coding. (Not included in the final submission so it is commented out in this notebook.)\n* local api：https://www.kaggle.com/code/jagofc/local-api/\n\nAs you can see from my code, I am a novice in machine learning and python.\nIf you see anything  to improve on or any mistakes, I'd be very happy to hear about them!","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gresearch_crypto\nimport time\nimport datetime\n\nimport pickle\nimport gc\n\nfrom tqdm import tqdm\n\nn_fold = 7\n\n# List for ensemble but not used due to inference time...\ninput_list = [\n    '../input/training-3rd-place-solution'\n]\nn_fold_list = [7]\n\nuse_supple_for_train = False\n\nTRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nSUPPLE_TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/supplemental_train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\n\npd.set_option('display.max_rows', 6)\npd.set_option('display.max_columns', 350)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:04.791034Z","iopub.execute_input":"2022-05-07T15:41:04.791589Z","iopub.status.idle":"2022-05-07T15:41:07.090244Z","shell.execute_reply.started":"2022-05-07T15:41:04.791421Z","shell.execute_reply":"2022-05-07T15:41:07.089419Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"models = [ [  [] for split in range(sum(n_fold_list))     ] for asset_id in range(14)]\n\nfor asset_id in range(14):\n    for input_number in range(len(input_list)):\n        for split in range(n_fold_list[input_number]):\n            models[asset_id][sum(n_fold_list[:input_number])+split] = pickle.load(open(f'{input_list[input_number]}/trained_model_id{asset_id}_fold{split}.pkl', 'rb'))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:07.092343Z","iopub.execute_input":"2022-05-07T15:41:07.092671Z","iopub.status.idle":"2022-05-07T15:41:08.547971Z","shell.execute_reply.started":"2022-05-07T15:41:07.092636Z","shell.execute_reply":"2022-05-07T15:41:08.547067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lags = [60,300,900]","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:08.549541Z","iopub.execute_input":"2022-05-07T15:41:08.549808Z","iopub.status.idle":"2022-05-07T15:41:08.553952Z","shell.execute_reply.started":"2022-05-07T15:41:08.549777Z","shell.execute_reply":"2022-05-07T15:41:08.553091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def reduce_mem_usage(df):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n\n    for col in df.columns:\n        col_type = df[col].dtype\n\n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:08.555533Z","iopub.execute_input":"2022-05-07T15:41:08.556027Z","iopub.status.idle":"2022-05-07T15:41:08.573907Z","shell.execute_reply.started":"2022-05-07T15:41:08.555982Z","shell.execute_reply":"2022-05-07T15:41:08.573242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset_details = pd.read_csv(ASSET_DETAILS_CSV).sort_values(\"Asset_ID\")\n# df_asset_details","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:08.576405Z","iopub.execute_input":"2022-05-07T15:41:08.577038Z","iopub.status.idle":"2022-05-07T15:41:08.614593Z","shell.execute_reply.started":"2022-05-07T15:41:08.576991Z","shell.execute_reply":"2022-05-07T15:41:08.613649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_features(df, train=True):   \n    if train == True:\n        totimestamp = lambda s: np.int32(time.mktime(datetime.datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\n        valid_window = [totimestamp(\"12/03/2021\")]\n#         valid_window = [totimestamp(\"15/08/2021\")]  #検証用\n        df['train_flg'] = np.where(df['timestamp']>=valid_window[0], 0,1)\n\n        supple_start_window = [totimestamp(\"22/09/2021\")]\n        if use_supple_for_train:\n            df['train_flg'] = np.where(df['timestamp']>=supple_start_window[0], 1 ,df['train_flg']  )\n\n   \n    for id in range(14):    \n        for lag in lags:\n            df[f'log_close/mean_{lag}_id{id}'] = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.append(np.convolve( np.array(df[f'Close_{id}']), np.ones(lag)/lag, mode=\"valid\"), np.ones(lag-1)), lag-1)  )\n            df[f'log_return_{lag}_id{id}']     = np.log( np.array(df[f'Close_{id}']) /  np.roll(np.array(df[f'Close_{id}']), lag)  )\n    for lag in lags:\n        df[f'mean_close/mean_{lag}'] =  np.mean(df.iloc[:,df.columns.str.startswith(f'log_close/mean_{lag}_id')], axis=1)\n        df[f'mean_log_returns_{lag}'] = np.mean(df.iloc[:,df.columns.str.startswith(f'log_return_{lag}_id')] ,    axis=1)\n        for id in range(14):\n            df[f'log_close/mean_{lag}-mean_close/mean_{lag}_id{id}'] = np.array( df[f'log_close/mean_{lag}_id{id}']) - np.array( df[f'mean_close/mean_{lag}']  )\n            df[f'log_return_{lag}-mean_log_returns_{lag}_id{id}']    = np.array( df[f'log_return_{lag}_id{id}'])     - np.array( df[f'mean_log_returns_{lag}'] )\n\n    if train == True:\n        for id in range(14):\n            df = df.drop([f'Close_{id}'], axis=1)\n        oldest_use_window = [totimestamp(\"12/01/2019\")]\n        df = df[  df['timestamp'] >= oldest_use_window[0]   ]\n\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:08.6158Z","iopub.execute_input":"2022-05-07T15:41:08.616017Z","iopub.status.idle":"2022-05-07T15:41:08.632972Z","shell.execute_reply.started":"2022-05-07T15:41:08.615991Z","shell.execute_reply":"2022-05-07T15:41:08.632118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n# generate feature column names\ndf_train = pd.read_csv(SUPPLE_TRAIN_CSV, usecols=['timestamp','Asset_ID', 'Close', 'Target'], nrows=max(lags)*20)\nprint(len(df_train['Asset_ID'].unique()))\ndf_train = reduce_mem_usage(df_train)\n# df_train\n\ntrain_merged = pd.DataFrame()\ntrain_merged[df_train.columns] = 0\nfor id in tqdm( range(14) ):\n    train_merged = train_merged.merge(df_train.loc[df_train[\"Asset_ID\"] == id, ['timestamp', 'Close','Target']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n        \ntrain_merged = train_merged.drop(df_train.columns.drop(\"timestamp\"), axis=1)\ndisplay(train_merged.head())\n\nnot_use_features_train = ['timestamp', 'train_flg']\nfor id in range(14):\n    not_use_features_train.append(f'Target_{id}')\n\nfeatures = get_features(train_merged, train=True).columns \nfeatures = features.drop(not_use_features_train)\nfeatures = list(features)\n# display(features)  \nlen(features)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:08.634084Z","iopub.execute_input":"2022-05-07T15:41:08.634365Z","iopub.status.idle":"2022-05-07T15:41:08.96465Z","shell.execute_reply.started":"2022-05-07T15:41:08.634334Z","shell.execute_reply":"2022-05-07T15:41:08.963654Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_merged\ndel df_train\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:08.96577Z","iopub.execute_input":"2022-05-07T15:41:08.965977Z","iopub.status.idle":"2022-05-07T15:41:09.096936Z","shell.execute_reply.started":"2022-05-07T15:41:08.96595Z","shell.execute_reply":"2022-05-07T15:41:09.095801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define max_lookback - an integer > (greater than) the furthest look back in your lagged features\nkeep_hist = max(lags)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:09.098525Z","iopub.execute_input":"2022-05-07T15:41:09.099044Z","iopub.status.idle":"2022-05-07T15:41:09.110034Z","shell.execute_reply.started":"2022-05-07T15:41:09.099009Z","shell.execute_reply":"2022-05-07T15:41:09.109251Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def merge_for_infer(df):\n    df_merged = pd.DataFrame()\n    df_merged[['timestamp', 'Asset_ID', 'Close']] = 0\n    for id in range(14):\n        df_merged = df_merged.merge(df.loc[df[\"Asset_ID\"] == id, ['timestamp', 'Close']].copy(), on=\"timestamp\", how='outer',suffixes=['', \"_\"+str(id)])\n \n    df_merged = df_merged.drop(['Asset_ID', 'Close'], axis=1)\n#     df_merged = df_merged.sort_values('timestamp', ascending=True)\n    return df_merged","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:09.111243Z","iopub.execute_input":"2022-05-07T15:41:09.111697Z","iopub.status.idle":"2022-05-07T15:41:09.122016Z","shell.execute_reply.started":"2022-05-07T15:41:09.111662Z","shell.execute_reply":"2022-05-07T15:41:09.121343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create dataframe to store data from the api to create lagged features\nhistory = pd.read_csv(SUPPLE_TRAIN_CSV, usecols=['timestamp','Asset_ID', 'Close'])\nhistory = history.tail(keep_hist*14)\n\nhistory = reduce_mem_usage(history)\nhistory_merged = merge_for_infer(history)\nhistory_merged = history_merged.tail(keep_hist)\nhistory_merged","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:09.123233Z","iopub.execute_input":"2022-05-07T15:41:09.123677Z","iopub.status.idle":"2022-05-07T15:41:15.800605Z","shell.execute_reply.started":"2022-05-07T15:41:09.123645Z","shell.execute_reply":"2022-05-07T15:41:15.799598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"one_line = history_merged.tail(1).copy()\n\ndef merge_infer_2(df, df_one_line):\n    for asset_id, close in zip(   df['Asset_ID'].values,  df['Close'].values   ): \n        df_one_line[f'Close_{asset_id}'] = close\n    return df_one_line","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:15.801898Z","iopub.execute_input":"2022-05-07T15:41:15.802216Z","iopub.status.idle":"2022-05-07T15:41:15.807979Z","shell.execute_reply.started":"2022-05-07T15:41:15.802184Z","shell.execute_reply":"2022-05-07T15:41:15.807138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"env = gresearch_crypto.make_env()\niter_test = env.iter_test()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:15.808939Z","iopub.execute_input":"2022-05-07T15:41:15.809572Z","iopub.status.idle":"2022-05-07T15:41:15.825619Z","shell.execute_reply.started":"2022-05-07T15:41:15.80954Z","shell.execute_reply":"2022-05-07T15:41:15.824597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# start = time.time()\n\n# for i, (df_test, df_pred) in enumerate(iter_test):\n#     df_test_merged = merge_infer_2(df_test, one_line)\n#     history_merged = pd.concat([history_merged, df_test_merged])\n#     x_test = get_features(history_merged, train=False)\n#     x_calc = x_test.iloc[-1]\n#     for j , (asset_id,row_id) in enumerate(  zip(   df_test['Asset_ID'].values,  df_test['row_id'].values   )   ): \n#         y_pred_list = []\n#         try:\n#             for split in range(n_fold):\n#                 y_pred_list.append(models[ asset_id ][split].predict(x_calc[features]))\n#             y_pred = np.median(y_pred_list)\n#         except Exception:\n#             y_pred = 0\n#         df_pred.loc[  df_pred['row_id'] == row_id ,  'Target'  ] = y_pred\n\n#     history_merged = history_merged.tail(keep_hist)\n    \n#     # Send submissions\n#     env.predict(df_pred)\n# stop = time.time()\n# print(stop-start)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:15.829352Z","iopub.execute_input":"2022-05-07T15:41:15.829978Z","iopub.status.idle":"2022-05-07T15:41:16.820719Z","shell.execute_reply.started":"2022-05-07T15:41:15.829929Z","shell.execute_reply":"2022-05-07T15:41:16.819242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# local test","metadata":{}},{"cell_type":"code","source":"# # Thanks to @jagofc\n# # https://www.kaggle.com/code/jagofc/local-api/\n\n# import local_api as la\n# train_df = la.read_csv_slice('../input/g-research-crypto-forecasting/train.csv')\n# # example_window = (la.datestring_to_timestamp(\"2021-07-02T00:00\"),\n# #                   la.datestring_to_timestamp(\"2021-08-17T05:00\"))\n# # la.LB_WINDOW\n# # api = la.API(train_df, use_window=example_window)\n# api = la.API(train_df, use_window=la.LB_WINDOW)\n\n# start_time = time.time()\n\n# for i, (df_test, df_pred) in enumerate(tqdm(api)):\n#     df_test_merged = merge_infer_2(df_test, one_line)    \n#     history_merged = pd.concat([history_merged, df_test_merged])    \n#     x_test = get_features(history_merged, False)\n#     x_calc = x_test.iloc[-1]\n    \n#     for j , (asset_id,row_id) in enumerate(  zip(   df_test['Asset_ID'].values,  df_test['row_id'].values   )   ): \n#         y_pred = 0       \n#         y_pred_list = []\n#         for split in range(n_fold):\n#             y_pred_list.append(models[ asset_id ][split].predict(x_calc[features]))\n#         y_pred = np.median(y_pred_list)     \n#         df_pred.loc[  df_pred['row_id'] == row_id ,  'Target'  ] = y_pred\n\n#     history_merged = history_merged.tail(keep_hist)\n    \n#     # Send submissions\n#     api.predict(df_pred)    \n# stop = time.time()\n# print(stop-start)\n\n# finish_time = time.time()\n\n# total_time = finish_time - start_time\n# iter_speed = api.init_num_times/total_time\n\n# print(f\"    num_fold = {n_fold}\")\n# print(f\"Iterations/s = {round(iter_speed, 3)}\")\n# print(f\"s/Iterations = {round(1/iter_speed, 3)}\")\n# test_iters = 60 * 24 * 100\n# print(f\"Expected number of iterations in test set is approx. {test_iters}\",\n#       f\"which will take {round(test_iters / (iter_speed * 3600), 2)} hours\",\n#       \"using this API emulator while making dummy predictions.\")\n\n# df, score = api.score()\n# print(f\"Your LB score is {round(score, 5)}\")","metadata":{"execution":{"iopub.status.busy":"2022-05-07T15:41:16.821951Z","iopub.status.idle":"2022-05-07T15:41:16.822523Z","shell.execute_reply.started":"2022-05-07T15:41:16.822219Z","shell.execute_reply":"2022-05-07T15:41:16.822246Z"},"trusted":true},"execution_count":null,"outputs":[]}]}