{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom pathlib import Path\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-30T13:54:05.125716Z","iopub.execute_input":"2021-12-30T13:54:05.126107Z","iopub.status.idle":"2021-12-30T13:54:05.141662Z","shell.execute_reply.started":"2021-12-30T13:54:05.126062Z","shell.execute_reply":"2021-12-30T13:54:05.140703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# debug mode setting\nis_dev = False\nnrows = 10000","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:54:05.143615Z","iopub.execute_input":"2021-12-30T13:54:05.144147Z","iopub.status.idle":"2021-12-30T13:54:05.147800Z","shell.execute_reply.started":"2021-12-30T13:54:05.144113Z","shell.execute_reply":"2021-12-30T13:54:05.147014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Description\nhttps://www.kaggle.com/c/g-research-crypto-forecasting/data?select=train.csv","metadata":{"execution":{"iopub.status.busy":"2021-11-14T05:53:39.463458Z","iopub.execute_input":"2021-11-14T05:53:39.463834Z","iopub.status.idle":"2021-11-14T05:53:39.470731Z","shell.execute_reply.started":"2021-11-14T05:53:39.463801Z","shell.execute_reply":"2021-11-14T05:53:39.469246Z"}}},{"cell_type":"markdown","source":"# train.csv","metadata":{"execution":{"iopub.status.busy":"2021-11-14T05:48:58.687229Z","iopub.execute_input":"2021-11-14T05:48:58.687619Z","iopub.status.idle":"2021-11-14T05:48:58.690754Z","shell.execute_reply.started":"2021-11-14T05:48:58.68759Z","shell.execute_reply":"2021-11-14T05:48:58.69001Z"}}},{"cell_type":"code","source":"# train.csv\ninput_dir = Path('../input/g-research-crypto-forecasting')\nif is_dev:\n    df_train = pd.read_csv(input_dir / 'train.csv', nrows=nrows)\nelse:\n    df_train = pd.read_csv(input_dir / 'train.csv')\ndf_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:54:05.149110Z","iopub.execute_input":"2021-12-30T13:54:05.149875Z","iopub.status.idle":"2021-12-30T13:55:08.622514Z","shell.execute_reply.started":"2021-12-30T13:54:05.149831Z","shell.execute_reply":"2021-12-30T13:55:08.621327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(df_train.columns)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:08.624800Z","iopub.execute_input":"2021-12-30T13:55:08.625204Z","iopub.status.idle":"2021-12-30T13:55:08.633623Z","shell.execute_reply.started":"2021-12-30T13:55:08.625146Z","shell.execute_reply":"2021-12-30T13:55:08.632357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:08.636087Z","iopub.execute_input":"2021-12-30T13:55:08.636322Z","iopub.status.idle":"2021-12-30T13:55:08.677732Z","shell.execute_reply.started":"2021-12-30T13:55:08.636295Z","shell.execute_reply":"2021-12-30T13:55:08.676896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check missing value\ndf_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:08.679267Z","iopub.execute_input":"2021-12-30T13:55:08.679604Z","iopub.status.idle":"2021-12-30T13:55:09.164902Z","shell.execute_reply.started":"2021-12-30T13:55:08.679558Z","shell.execute_reply":"2021-12-30T13:55:09.164046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# example_test.csv","metadata":{}},{"cell_type":"code","source":"df_test = pd.read_csv(input_dir / 'example_test.csv')\ndf_test.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.166267Z","iopub.execute_input":"2021-12-30T13:55:09.166491Z","iopub.status.idle":"2021-12-30T13:55:09.181904Z","shell.execute_reply.started":"2021-12-30T13:55:09.166464Z","shell.execute_reply":"2021-12-30T13:55:09.180751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.183180Z","iopub.execute_input":"2021-12-30T13:55:09.183442Z","iopub.status.idle":"2021-12-30T13:55:09.200608Z","shell.execute_reply.started":"2021-12-30T13:55:09.183400Z","shell.execute_reply":"2021-12-30T13:55:09.199775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.group_num.value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.201944Z","iopub.execute_input":"2021-12-30T13:55:09.202224Z","iopub.status.idle":"2021-12-30T13:55:09.222138Z","shell.execute_reply.started":"2021-12-30T13:55:09.202194Z","shell.execute_reply":"2021-12-30T13:55:09.221244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# example_sample_submission","metadata":{}},{"cell_type":"code","source":"df_sample_submission = pd.read_csv(input_dir / 'example_sample_submission.csv')\ndf_sample_submission.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.223639Z","iopub.execute_input":"2021-12-30T13:55:09.223962Z","iopub.status.idle":"2021-12-30T13:55:09.242076Z","shell.execute_reply.started":"2021-12-30T13:55:09.223914Z","shell.execute_reply":"2021-12-30T13:55:09.241011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.243332Z","iopub.execute_input":"2021-12-30T13:55:09.243645Z","iopub.status.idle":"2021-12-30T13:55:09.256571Z","shell.execute_reply.started":"2021-12-30T13:55:09.243604Z","shell.execute_reply":"2021-12-30T13:55:09.255421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# asset_details.csv","metadata":{}},{"cell_type":"code","source":"df_asset = pd.read_csv(input_dir / 'asset_details.csv')\ndf_asset.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.258120Z","iopub.execute_input":"2021-12-30T13:55:09.258835Z","iopub.status.idle":"2021-12-30T13:55:09.274372Z","shell.execute_reply.started":"2021-12-30T13:55:09.258788Z","shell.execute_reply":"2021-12-30T13:55:09.273353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_asset.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.277484Z","iopub.execute_input":"2021-12-30T13:55:09.278514Z","iopub.status.idle":"2021-12-30T13:55:09.288647Z","shell.execute_reply.started":"2021-12-30T13:55:09.278472Z","shell.execute_reply":"2021-12-30T13:55:09.288010Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# supplemental_train.csv","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:30:14.378103Z","iopub.execute_input":"2021-11-14T06:30:14.378419Z","iopub.status.idle":"2021-11-14T06:30:14.38248Z","shell.execute_reply.started":"2021-11-14T06:30:14.378389Z","shell.execute_reply":"2021-11-14T06:30:14.381809Z"}}},{"cell_type":"code","source":"if is_dev:\n    df_sup_train = pd.read_csv(input_dir / 'supplemental_train.csv', nrows=nrows)\nelse:\n    df_sup_train = pd.read_csv(input_dir / 'supplemental_train.csv')\ndf_sup_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:09.290159Z","iopub.execute_input":"2021-12-30T13:55:09.290400Z","iopub.status.idle":"2021-12-30T13:55:14.810997Z","shell.execute_reply.started":"2021-12-30T13:55:09.290361Z","shell.execute_reply":"2021-12-30T13:55:14.810082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sup_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:14.812652Z","iopub.execute_input":"2021-12-30T13:55:14.813487Z","iopub.status.idle":"2021-12-30T13:55:14.828982Z","shell.execute_reply.started":"2021-12-30T13:55:14.813437Z","shell.execute_reply":"2021-12-30T13:55:14.828299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_sup_train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:14.830204Z","iopub.execute_input":"2021-12-30T13:55:14.830756Z","iopub.status.idle":"2021-12-30T13:55:14.892170Z","shell.execute_reply.started":"2021-12-30T13:55:14.830718Z","shell.execute_reply":"2021-12-30T13:55:14.891548Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Merge train.csv and asset_details.csv","metadata":{}},{"cell_type":"code","source":"df_train = df_train.merge(df_asset.drop('Weight', axis=1), on='Asset_ID')\ndf_train['timestamp'] = df_train['timestamp'].astype('datetime64[s]')\ndf_train = df_train.sort_values(['timestamp'])\ndf_train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:14.893289Z","iopub.execute_input":"2021-12-30T13:55:14.893519Z","iopub.status.idle":"2021-12-30T13:55:25.603246Z","shell.execute_reply.started":"2021-12-30T13:55:14.893492Z","shell.execute_reply":"2021-12-30T13:55:25.602290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Plot data","metadata":{"execution":{"iopub.status.busy":"2021-11-14T06:39:51.648821Z","iopub.execute_input":"2021-11-14T06:39:51.649171Z","iopub.status.idle":"2021-11-14T06:39:51.652772Z","shell.execute_reply.started":"2021-11-14T06:39:51.649135Z","shell.execute_reply":"2021-11-14T06:39:51.652058Z"}}},{"cell_type":"code","source":"binance_coin = df_train.query(f'Asset_ID == 0')\nbit_coin = df_train.query(f'Asset_ID == 1')\nbit_cash = df_train.query(f'Asset_ID == 2')\neos_io = df_train.query(f'Asset_ID == 5')\neth_classic = df_train.query(f'Asset_ID == 7')\n\ncurrencies = {'binance_coin': binance_coin,\n              'bit_coin': bit_coin,\n              'bit_cash': bit_cash,\n              'eos_io': eos_io,\n              'eth_classic': eth_classic\n             }","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:25.604570Z","iopub.execute_input":"2021-12-30T13:55:25.604838Z","iopub.status.idle":"2021-12-30T13:55:28.130151Z","shell.execute_reply.started":"2021-12-30T13:55:25.604807Z","shell.execute_reply":"2021-12-30T13:55:28.129275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\nfrom matplotlib import pyplot as plt\nplt.rcParams[\"font.size\"] = 15","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:28.131610Z","iopub.execute_input":"2021-12-30T13:55:28.131928Z","iopub.status.idle":"2021-12-30T13:55:29.104408Z","shell.execute_reply.started":"2021-12-30T13:55:28.131888Z","shell.execute_reply":"2021-12-30T13:55:29.103523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_lines(df, index, columns, title, tail=None):\n    if tail is not None:\n        df_plot = df.copy()[-tail:]\n    else:\n        df_plot = df.copy()\n    df_plot = df_plot.set_index(index)[columns]\n    plt.figure(figsize=(20, 10))\n    if title is not None:\n        plt.title(title)\n    sns.lineplot(data=df_plot)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:29.105521Z","iopub.execute_input":"2021-12-30T13:55:29.105737Z","iopub.status.idle":"2021-12-30T13:55:29.112612Z","shell.execute_reply.started":"2021-12-30T13:55:29.105712Z","shell.execute_reply":"2021-12-30T13:55:29.111721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = 'timestamp'\ncolumns = ['Target']\nplot_lines(bit_coin, index, columns, 'bit_coin', tail=100000)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:29.113980Z","iopub.execute_input":"2021-12-30T13:55:29.114222Z","iopub.status.idle":"2021-12-30T13:55:34.389816Z","shell.execute_reply.started":"2021-12-30T13:55:29.114194Z","shell.execute_reply":"2021-12-30T13:55:34.388910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"index = 'timestamp'\ncolumns = ['High', 'Open', 'Close', 'Low']\nplot_lines(bit_coin, index, columns, 'bit_coin', tail=1000)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:34.391344Z","iopub.execute_input":"2021-12-30T13:55:34.391687Z","iopub.status.idle":"2021-12-30T13:55:35.114744Z","shell.execute_reply.started":"2021-12-30T13:55:34.391643Z","shell.execute_reply":"2021-12-30T13:55:35.114095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_line_dict(dict_df, index, columns, tail=None):\n    i = int(np.ceil(len(dict_df) / 2))\n    j = 2\n    fig, axes = plt.subplots(i, j, figsize=(40, 40))\n    keys = dict_df.keys()\n    for idx, key in enumerate(keys):\n        df = dict_df[key]\n        if tail is not None:\n            df_plot = df.copy()[-tail:]\n        else:\n            df_plot = df.copy()\n        df_plot = df_plot.set_index(index)[columns]\n        ax = axes[idx // j][idx % j]\n        ax.set_title(key)\n        sns.lineplot(data=df_plot, ax=ax)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:35.116011Z","iopub.execute_input":"2021-12-30T13:55:35.116658Z","iopub.status.idle":"2021-12-30T13:55:35.124325Z","shell.execute_reply.started":"2021-12-30T13:55:35.116619Z","shell.execute_reply":"2021-12-30T13:55:35.123406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_line_dict(currencies, index, columns, tail=1000)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:35.125469Z","iopub.execute_input":"2021-12-30T13:55:35.125797Z","iopub.status.idle":"2021-12-30T13:55:38.621212Z","shell.execute_reply.started":"2021-12-30T13:55:35.125767Z","shell.execute_reply":"2021-12-30T13:55:38.620435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Volume","metadata":{"execution":{"iopub.status.busy":"2021-11-14T08:41:37.577956Z","iopub.execute_input":"2021-11-14T08:41:37.578307Z","iopub.status.idle":"2021-11-14T08:41:37.582324Z","shell.execute_reply.started":"2021-11-14T08:41:37.578271Z","shell.execute_reply":"2021-11-14T08:41:37.58163Z"}}},{"cell_type":"code","source":"def plot_line_pivot(df, index, divide_col, value, tail=None):\n    if tail is not None:\n        df = df[-tail:]\n    df = pd.pivot_table(df, index=index, columns=[divide_col], values=value)\n    plt.figure(figsize=(20, 10))\n    sns.lineplot(data=df)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:38.622388Z","iopub.execute_input":"2021-12-30T13:55:38.622761Z","iopub.status.idle":"2021-12-30T13:55:38.628666Z","shell.execute_reply.started":"2021-12-30T13:55:38.622705Z","shell.execute_reply":"2021-12-30T13:55:38.627738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_line_pivot(df_train, index, divide_col='Asset_Name', value='Volume', tail=10000)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:38.629824Z","iopub.execute_input":"2021-12-30T13:55:38.630094Z","iopub.status.idle":"2021-12-30T13:55:39.746897Z","shell.execute_reply.started":"2021-12-30T13:55:38.630063Z","shell.execute_reply":"2021-12-30T13:55:39.746270Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Amount","metadata":{}},{"cell_type":"code","source":"df_train['Estimated_Amount'] = df_train['Volume'] * df_train['Close']","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:39.748032Z","iopub.execute_input":"2021-12-30T13:55:39.748694Z","iopub.status.idle":"2021-12-30T13:55:39.867896Z","shell.execute_reply.started":"2021-12-30T13:55:39.748658Z","shell.execute_reply":"2021-12-30T13:55:39.866943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.groupby('Asset_Name')['Estimated_Amount'].mean().astype(int).sort_values(ascending=False).plot.bar()","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:39.869256Z","iopub.execute_input":"2021-12-30T13:55:39.869516Z","iopub.status.idle":"2021-12-30T13:55:43.083644Z","shell.execute_reply.started":"2021-12-30T13:55:39.869485Z","shell.execute_reply":"2021-12-30T13:55:43.082323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_line_pivot(df_train, index, divide_col='Asset_Name', value='Estimated_Amount', tail=10000)","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:43.084829Z","iopub.execute_input":"2021-12-30T13:55:43.085105Z","iopub.status.idle":"2021-12-30T13:55:44.326358Z","shell.execute_reply.started":"2021-12-30T13:55:43.085072Z","shell.execute_reply":"2021-12-30T13:55:44.325391Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Correlations between Assets","metadata":{"execution":{"iopub.status.busy":"2021-11-14T08:34:04.491995Z","iopub.execute_input":"2021-11-14T08:34:04.492509Z","iopub.status.idle":"2021-11-14T08:34:04.496096Z","shell.execute_reply.started":"2021-11-14T08:34:04.492457Z","shell.execute_reply":"2021-11-14T08:34:04.495203Z"}}},{"cell_type":"code","source":"df_train_pivot = df_train.pivot_table(index=['timestamp'], columns=['Asset_Name'], values='Target', aggfunc='max')","metadata":{"execution":{"iopub.status.busy":"2021-12-30T13:55:44.327790Z","iopub.execute_input":"2021-12-30T13:55:44.328172Z","iopub.status.idle":"2021-12-30T13:56:06.741995Z","shell.execute_reply.started":"2021-12-30T13:55:44.328132Z","shell.execute_reply":"2021-12-30T13:56:06.741086Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train_pivot","metadata":{"execution":{"iopub.status.busy":"2021-12-30T14:02:51.101237Z","iopub.execute_input":"2021-12-30T14:02:51.101545Z","iopub.status.idle":"2021-12-30T14:02:51.131318Z","shell.execute_reply.started":"2021-12-30T14:02:51.101513Z","shell.execute_reply":"2021-12-30T14:02:51.130449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tail_num = 1000\narray_scatter_matrix = pd.plotting.scatter_matrix(df_train_pivot.fillna(0).tail(tail_num), alpha=0.2, figsize=(30, 30))","metadata":{"execution":{"iopub.status.busy":"2021-12-30T14:03:18.646746Z","iopub.execute_input":"2021-12-30T14:03:18.647515Z","iopub.status.idle":"2021-12-30T14:03:33.152063Z","shell.execute_reply.started":"2021-12-30T14:03:18.647453Z","shell.execute_reply":"2021-12-30T14:03:33.151269Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = df_train_pivot.corr()\nmask = np.zeros_like(corr)\nmask[np.triu_indices_from(mask)] = True\nwith sns.axes_style(\"white\"):\n    f, ax = plt.subplots(figsize=(20, 15))\n    ax = sns.heatmap(corr, mask=mask, square=True, annot=True, cmap=\"YlGnBu\")","metadata":{"execution":{"iopub.status.busy":"2021-12-30T14:03:55.816127Z","iopub.execute_input":"2021-12-30T14:03:55.816917Z","iopub.status.idle":"2021-12-30T14:03:58.019844Z","shell.execute_reply.started":"2021-12-30T14:03:55.816876Z","shell.execute_reply":"2021-12-30T14:03:58.018861Z"},"trusted":true},"execution_count":null,"outputs":[]}]}