{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Crypto_DataComplement_Plot\n- Plot the Target and Close for each cryptocurrency.\n- Complement each cryptocurrency data so that they have the same time period.","metadata":{}},{"cell_type":"code","source":"import os\nimport random\nimport pandas as pd\nimport numpy as np\nfrom lightgbm import LGBMRegressor\nimport gresearch_crypto\nfrom tqdm.notebook import trange, tqdm\nimport matplotlib.pyplot as plt\nimport gresearch_crypto","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:11:12.658135Z","iopub.execute_input":"2021-11-22T13:11:12.658500Z","iopub.status.idle":"2021-11-22T13:11:15.031582Z","shell.execute_reply.started":"2021-11-22T13:11:12.658408Z","shell.execute_reply":"2021-11-22T13:11:15.030802Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"TRAIN_CSV = '/kaggle/input/g-research-crypto-forecasting/train.csv'\nASSET_DETAILS_CSV = '/kaggle/input/g-research-crypto-forecasting/asset_details.csv'\nSEED = 2021\nREMOVE_LB_TEST_OVERLAPPING_DATA = True\n\ndef fix_seeds(seed):\n    np.random.seed(seed)\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nfix_seeds(SEED)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:11:15.033231Z","iopub.execute_input":"2021-11-22T13:11:15.033459Z","iopub.status.idle":"2021-11-22T13:11:15.038862Z","shell.execute_reply.started":"2021-11-22T13:11:15.033433Z","shell.execute_reply":"2021-11-22T13:11:15.037979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(TRAIN_CSV)\nasset_details_df = pd.read_csv(ASSET_DETAILS_CSV).sort_values('Asset_ID')\n\n# Remove the future\nif REMOVE_LB_TEST_OVERLAPPING_DATA:\n    train_df['datetime'] = pd.to_datetime(train_df['timestamp'], unit='s')\n    train_df = train_df[train_df['datetime'] < '2021-06-13 00:00:00']\n\ndisplay(train_df.head(3))\ndisplay(asset_details_df.head(3))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:11:15.040030Z","iopub.execute_input":"2021-11-22T13:11:15.040267Z","iopub.status.idle":"2021-11-22T13:12:17.197249Z","shell.execute_reply.started":"2021-11-22T13:11:15.040235Z","shell.execute_reply":"2021-11-22T13:12:17.196093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data complement","metadata":{"execution":{"iopub.status.busy":"2021-11-22T08:54:58.670169Z","iopub.execute_input":"2021-11-22T08:54:58.670618Z","iopub.status.idle":"2021-11-22T08:54:58.676171Z","shell.execute_reply.started":"2021-11-22T08:54:58.67057Z","shell.execute_reply":"2021-11-22T08:54:58.675124Z"}}},{"cell_type":"code","source":"def data_complement(train_df, asset_details_df):\n    asset_df_list = []\n    for asset_id in tqdm(asset_details_df.Asset_ID):\n        \n        asset_name = asset_details_df.loc[asset_details_df.Asset_ID==asset_id, 'Asset_Name'].values[0]\n        print(f\"{asset_id: >2} {asset_name: <20}\", end=' ')\n        asset_df = train_df.loc[train_df.Asset_ID == asset_id].set_index('timestamp')\n        print(f\"Length : {len(asset_df):,}\", end=' -> ')\n        \n        # Re Index\n        asset_df = asset_df.reindex(range(train_df.timestamp.min(),train_df.timestamp.max()+60,60), method='pad')\n        asset_df['datetime'] = pd.to_datetime(asset_df.index, unit='s')\n        \n        # Data complement\n        asset_df['Asset_ID'] = asset_id\n        fill_cols = ['Count', 'Open', 'High', 'Low', 'Close', 'Volume', 'VWAP']\n        asset_df[fill_cols] = asset_df[fill_cols].replace([np.inf, -np.inf], np.nan)  # Replace Infinite\n        asset_df[fill_cols] = asset_df[fill_cols].interpolate('index')\n        asset_df['Target'] =  asset_df['Target'].fillna(value=0.0)\n        asset_df[fill_cols] =  asset_df[fill_cols].fillna(asset_df[fill_cols].mean())\n        \n        # Drop if nan\n        asset_df = asset_df.dropna(subset=['Target'])\n        \n        print(f\"{len(asset_df):,}\")\n        asset_df_list.append(asset_df)  \n        \n    return pd.concat(asset_df_list, axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:12:17.199709Z","iopub.execute_input":"2021-11-22T13:12:17.200063Z","iopub.status.idle":"2021-11-22T13:12:17.213471Z","shell.execute_reply.started":"2021-11-22T13:12:17.200019Z","shell.execute_reply":"2021-11-22T13:12:17.212636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_complemented_df = data_complement(train_df, asset_details_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:12:17.214712Z","iopub.execute_input":"2021-11-22T13:12:17.215009Z","iopub.status.idle":"2021-11-22T13:12:49.009369Z","shell.execute_reply.started":"2021-11-22T13:12:17.214977Z","shell.execute_reply":"2021-11-22T13:12:49.008585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data view","metadata":{}},{"cell_type":"code","source":"def plt_assets(df, asset_details_df):\n    plt.subplots_adjust(wspace=10.0, hspace=2.0)\n    fig = plt.figure(figsize=(18, 25))\n    for i, asset_id in enumerate(asset_details_df.Asset_ID):\n        asset_name = asset_details_df.loc[asset_details_df.Asset_ID==asset_id, 'Asset_Name'].values[0]\n        ax = fig.add_subplot(7, 2, i+1)\n        plt.xlabel('DateTime')\n        plt.ylabel(asset_name)\n        ax.plot(df.loc[df.Asset_ID==asset_id, 'Close'], color='red', label='Close')\n        plt.legend(loc='upper left')\n        ax2 = ax.twinx()\n        plt.grid()\n        ax2.plot(df.loc[df.Asset_ID==asset_id, 'Target'], color='blue', alpha=0.5, label='Target')\n        plt.legend(loc='upper right')\n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:22:50.488965Z","iopub.execute_input":"2021-11-22T13:22:50.489244Z","iopub.status.idle":"2021-11-22T13:22:50.498681Z","shell.execute_reply.started":"2021-11-22T13:22:50.489213Z","shell.execute_reply":"2021-11-22T13:22:50.497631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt_assets(train_complemented_df, asset_details_df)","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:22:50.704060Z","iopub.execute_input":"2021-11-22T13:22:50.704995Z","iopub.status.idle":"2021-11-22T13:23:01.255702Z","shell.execute_reply.started":"2021-11-22T13:22:50.704918Z","shell.execute_reply":"2021-11-22T13:23:01.254933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display(train_complemented_df.head(3))","metadata":{"execution":{"iopub.status.busy":"2021-11-22T13:13:12.906432Z","iopub.execute_input":"2021-11-22T13:13:12.906710Z","iopub.status.idle":"2021-11-22T13:13:12.925392Z","shell.execute_reply.started":"2021-11-22T13:13:12.906672Z","shell.execute_reply":"2021-11-22T13:13:12.924418Z"},"trusted":true},"execution_count":null,"outputs":[]}]}