{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Objective :-\nHere we will replace the Nan values in the 'Target' column of 'train.csv' with the recreated targets present here https://www.kaggle.com/swaralipibose/target-recreation-but-its-99-percent-accurate so that we can gather up more data it has also been discussed here https://www.kaggle.com/c/g-research-crypto-forecasting/discussion/293654 which has proposed this method","metadata":{}},{"cell_type":"markdown","source":"# Import","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\ndf=pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ndf.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-10T05:26:24.823265Z","iopub.execute_input":"2021-12-10T05:26:24.824118Z","iopub.status.idle":"2021-12-10T05:27:20.268653Z","shell.execute_reply.started":"2021-12-10T05:26:24.824045Z","shell.execute_reply":"2021-12-10T05:27:20.267694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"details=pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\ndetails.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:27:20.270764Z","iopub.execute_input":"2021-12-10T05:27:20.271093Z","iopub.status.idle":"2021-12-10T05:27:20.288247Z","shell.execute_reply.started":"2021-12-10T05:27:20.271041Z","shell.execute_reply":"2021-12-10T05:27:20.287204Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Percentage of Nan values","metadata":{"execution":{"iopub.status.busy":"2021-12-08T15:17:06.83466Z","iopub.execute_input":"2021-12-08T15:17:06.835258Z","iopub.status.idle":"2021-12-08T15:17:06.934906Z","shell.execute_reply.started":"2021-12-08T15:17:06.835206Z","shell.execute_reply":"2021-12-08T15:17:06.933989Z"}}},{"cell_type":"code","source":"import numpy as np\n\nfor i in range(14):\n \n    dfcrop=df[df['Asset_ID']==i]\n    print('Percentage of values not nan',(1-(np.sum((dfcrop['Target'].isnull()).astype(int))/dfcrop.shape[0]))*100)\n  \n    \n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:27:20.289926Z","iopub.execute_input":"2021-12-10T05:27:20.290614Z","iopub.status.idle":"2021-12-10T05:27:24.215569Z","shell.execute_reply.started":"2021-12-10T05:27:20.290564Z","shell.execute_reply":"2021-12-10T05:27:24.214357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Recreate Targets","metadata":{}},{"cell_type":"code","source":"from tqdm import tqdm\ndef compute(df):\n    \n    R=list()\n    c=list(df['Close'])\n    for i in range(df.shape[0]):\n        future=c[min([i+16,df.shape[0]-1])]\n        past=c[min([i+1,df.shape[0]-1])]\n        R.append(future/past)\n    R=np.array(R)\n\n    df['pred']=R-1\n    return df\ncrops=list()\nfor a in tqdm(list(range(14))):\n    \n    crops.append(compute(df[df['Asset_ID']==a]))\n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:27:24.218539Z","iopub.execute_input":"2021-12-10T05:27:24.218966Z","iopub.status.idle":"2021-12-10T05:28:54.748138Z","shell.execute_reply.started":"2021-12-10T05:27:24.218904Z","shell.execute_reply":"2021-12-10T05:28:54.747101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nconc=pd.concat(crops)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:28:54.749690Z","iopub.execute_input":"2021-12-10T05:28:54.749899Z","iopub.status.idle":"2021-12-10T05:28:55.847799Z","shell.execute_reply.started":"2021-12-10T05:28:54.749873Z","shell.execute_reply":"2021-12-10T05:28:55.846899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conc.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:28:55.849405Z","iopub.execute_input":"2021-12-10T05:28:55.850564Z","iopub.status.idle":"2021-12-10T05:28:55.869164Z","shell.execute_reply.started":"2021-12-10T05:28:55.850515Z","shell.execute_reply":"2021-12-10T05:28:55.868197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conc=conc.reset_index()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:28:55.870838Z","iopub.execute_input":"2021-12-10T05:28:55.871144Z","iopub.status.idle":"2021-12-10T05:29:02.174466Z","shell.execute_reply.started":"2021-12-10T05:28:55.871104Z","shell.execute_reply":"2021-12-10T05:29:02.173438Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"j=np.array(list(conc['Target'].isnull()))\nnew_targets=np.where(j,conc['pred'],conc['Target'])","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:02.175730Z","iopub.execute_input":"2021-12-10T05:29:02.176744Z","iopub.status.idle":"2021-12-10T05:29:08.568719Z","shell.execute_reply.started":"2021-12-10T05:29:02.176705Z","shell.execute_reply":"2021-12-10T05:29:08.567729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"conc['Target']=new_targets\nconc=conc.drop(columns=['pred'],axis=1)\nconc.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:08.570194Z","iopub.execute_input":"2021-12-10T05:29:08.570512Z","iopub.status.idle":"2021-12-10T05:29:10.000214Z","shell.execute_reply.started":"2021-12-10T05:29:08.570472Z","shell.execute_reply":"2021-12-10T05:29:09.999382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# No more nan values ","metadata":{}},{"cell_type":"code","source":"import numpy as np\n\nfor i in range(14):\n    dfcrop=conc[conc['Asset_ID']==i]\n    print('Percentage of values not nan',(1-(np.sum((dfcrop['Target'].isnull()).astype(int))/dfcrop.shape[0]))*100)\n  \n    \n","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:10.002618Z","iopub.execute_input":"2021-12-10T05:29:10.002871Z","iopub.status.idle":"2021-12-10T05:29:11.538654Z","shell.execute_reply.started":"2021-12-10T05:29:10.002837Z","shell.execute_reply":"2021-12-10T05:29:11.537862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save new train.h5 file h5 is way faster","metadata":{}},{"cell_type":"code","source":"conc.to_hdf('train.h5',key='conc', mode='w',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:11.539679Z","iopub.execute_input":"2021-12-10T05:29:11.539893Z","iopub.status.idle":"2021-12-10T05:29:21.018965Z","shell.execute_reply.started":"2021-12-10T05:29:11.539868Z","shell.execute_reply":"2021-12-10T05:29:21.018295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del conc,crops","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:21.022534Z","iopub.execute_input":"2021-12-10T05:29:21.025055Z","iopub.status.idle":"2021-12-10T05:29:21.051488Z","shell.execute_reply.started":"2021-12-10T05:29:21.025008Z","shell.execute_reply":"2021-12-10T05:29:21.047428Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load your saved h5 file","metadata":{}},{"cell_type":"code","source":"df=pd.read_hdf('train.h5')\ndf=df.drop(columns=['index'])\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:21.056159Z","iopub.execute_input":"2021-12-10T05:29:21.057136Z","iopub.status.idle":"2021-12-10T05:29:38.157638Z","shell.execute_reply.started":"2021-12-10T05:29:21.057086Z","shell.execute_reply":"2021-12-10T05:29:38.156783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-10T05:29:38.158777Z","iopub.execute_input":"2021-12-10T05:29:38.159011Z","iopub.status.idle":"2021-12-10T05:29:38.164678Z","shell.execute_reply.started":"2021-12-10T05:29:38.158983Z","shell.execute_reply":"2021-12-10T05:29:38.163900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Thank you","metadata":{}}]}