{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Speed-up data loading!**","metadata":{}},{"cell_type":"markdown","source":"Train.csv is a big file that takes some time to load. I created this notebook to have a feather version of the file which will be much faster to load. You don't need to run this notebook. If you want, you can just use the output as your input and speed-up the data loading **from one minute to one second!**\n\nTo load the feather file, just add data from this notebook and open the file with <code>pd.read_feather('../input/speed-up-data-loading/train.feather')</code>\n\nI changed the datatypes of the DataFrame (I took most of the code from [this](https://www.kaggle.com/toomuchsauce/g-crypto-interactive-dashboard-indicators) notebook) to reduce the size of the dataset. If you want to use the original data types, you can skip the call to reduce_memory function.\n\nAt the end of the notebook I fill the gaps to have all the assets for all the existing timestamps. This is convenient for me, anyone not interested in this format can ignore the <code>transformed_train.feather</code> file and just use <code>train.feather</code>","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom datetime import datetime","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-08T16:35:12.51514Z","iopub.execute_input":"2021-12-08T16:35:12.515695Z","iopub.status.idle":"2021-12-08T16:35:12.519057Z","shell.execute_reply.started":"2021-12-08T16:35:12.515659Z","shell.execute_reply":"2021-12-08T16:35:12.518405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_time = datetime.timestamp(datetime.now())\ncrypto_df = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\nprint(\"load in\",datetime.timestamp(datetime.now())-start_time,\"seconds\")","metadata":{"execution":{"iopub.status.busy":"2021-12-08T16:37:50.89831Z","iopub.execute_input":"2021-12-08T16:37:50.8989Z","iopub.status.idle":"2021-12-08T16:38:46.507482Z","shell.execute_reply.started":"2021-12-08T16:37:50.89886Z","shell.execute_reply":"2021-12-08T16:38:46.506453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T16:41:42.433216Z","iopub.execute_input":"2021-12-08T16:41:42.433556Z","iopub.status.idle":"2021-12-08T16:41:42.45038Z","shell.execute_reply.started":"2021-12-08T16:41:42.433518Z","shell.execute_reply":"2021-12-08T16:41:42.449518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/toomuchsauce/g-crypto-interactive-dashboard-indicators\n\ndef reduce_memory(df):\n    \n    before = df.memory_usage().sum()  \n    \n    for col in df.columns:        \n        dtype = df[col].dtype\n        if dtype == 'float64':\n            c_min = df[col].min()\n            c_max = df[col].max()        \n            if c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                df[col] = df[col].astype(np.float32)\n            else:\n                df[col] = df[col].astype(np.float64)\n\n    df['Asset_ID'] = df['Asset_ID'].astype('int8')\n    df['Count'] = df['Count'].astype('int32')\n    df['timestamp'] = df['timestamp'].astype('uint32')\n                    \n    after = df.memory_usage().sum()\n    \n    print('Memory taken before transformation : ', before)\n    print('Memory taken after transformation : ', after)\n    print('Memory taken reduced by : ',( before - after) * 100/ before, '%')\n    \n    return df\n\ncrypto_df = reduce_memory(crypto_df)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T16:39:36.330193Z","iopub.execute_input":"2021-12-08T16:39:36.330543Z","iopub.status.idle":"2021-12-08T16:39:39.966482Z","shell.execute_reply.started":"2021-12-08T16:39:36.330512Z","shell.execute_reply":"2021-12-08T16:39:39.965564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.to_feather('train.feather')","metadata":{"execution":{"iopub.status.busy":"2021-12-08T16:39:54.253285Z","iopub.execute_input":"2021-12-08T16:39:54.254094Z","iopub.status.idle":"2021-12-08T16:39:57.027903Z","shell.execute_reply.started":"2021-12-08T16:39:54.254039Z","shell.execute_reply":"2021-12-08T16:39:57.026982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#del crypto_df\nstart_time = datetime.timestamp(datetime.now())\ncrypto_df = pd.read_feather(\"train.feather\")\nprint(\"load in\",datetime.timestamp(datetime.now())-start_time,\"seconds\")","metadata":{"execution":{"iopub.status.busy":"2021-12-08T16:40:59.061788Z","iopub.execute_input":"2021-12-08T16:40:59.062205Z","iopub.status.idle":"2021-12-08T16:41:00.635132Z","shell.execute_reply.started":"2021-12-08T16:40:59.062175Z","shell.execute_reply":"2021-12-08T16:41:00.634613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-12-08T16:41:18.892832Z","iopub.execute_input":"2021-12-08T16:41:18.89381Z","iopub.status.idle":"2021-12-08T16:41:18.915877Z","shell.execute_reply.started":"2021-12-08T16:41:18.893766Z","shell.execute_reply":"2021-12-08T16:41:18.915069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Transform data to fill gaps**","metadata":{}},{"cell_type":"code","source":"df1 = pd.DataFrame({'timestamp':crypto_df.timestamp.unique()}).sort_values(['timestamp'])\ndf2 = pd.DataFrame({'Asset_ID':crypto_df.Asset_ID.unique()}).sort_values(['Asset_ID'])\ndf_x = df1.merge(df2, how='cross').set_index(['timestamp','Asset_ID'],drop=True)\ncrypto_df=df_x.join(crypto_df.set_index(['timestamp','Asset_ID'],drop=True).sort_index()[['Open','High','Low','Close','Volume','VWAP','Target']],how='left').fillna(0).reset_index()\ndel df1\ndel df2\ndel df_x\ncrypto_df.to_feather('transformed_train.feather')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"crypto_df.head(10)","metadata":{},"execution_count":null,"outputs":[]}]}