{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"### Support the notebook if you like it. Upvote is FREE :)","metadata":{}},{"cell_type":"markdown","source":"# Imports","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-02T03:07:16.365867Z","iopub.execute_input":"2021-12-02T03:07:16.366231Z","iopub.status.idle":"2021-12-02T03:07:17.301354Z","shell.execute_reply.started":"2021-12-02T03:07:16.366133Z","shell.execute_reply":"2021-12-02T03:07:17.30061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read data","metadata":{}},{"cell_type":"code","source":"assets = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\nsup_train = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')\nsample = pd.read_csv('../input/g-research-crypto-forecasting/example_sample_submission.csv')\nex_test = pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:07:17.302733Z","iopub.execute_input":"2021-12-02T03:07:17.302963Z","iopub.status.idle":"2021-12-02T03:07:22.150638Z","shell.execute_reply.started":"2021-12-02T03:07:17.302936Z","shell.execute_reply":"2021-12-02T03:07:22.149921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:07:22.15175Z","iopub.execute_input":"2021-12-02T03:07:22.152259Z","iopub.status.idle":"2021-12-02T03:08:13.759542Z","shell.execute_reply.started":"2021-12-02T03:07:22.152225Z","shell.execute_reply":"2021-12-02T03:08:13.758954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA","metadata":{}},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:13.76099Z","iopub.execute_input":"2021-12-02T03:08:13.761501Z","iopub.status.idle":"2021-12-02T03:08:13.785851Z","shell.execute_reply.started":"2021-12-02T03:08:13.761469Z","shell.execute_reply":"2021-12-02T03:08:13.783903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:13.787103Z","iopub.execute_input":"2021-12-02T03:08:13.787389Z","iopub.status.idle":"2021-12-02T03:08:13.792906Z","shell.execute_reply.started":"2021-12-02T03:08:13.787349Z","shell.execute_reply":"2021-12-02T03:08:13.792139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We have 24 Million records with 10 columns","metadata":{}},{"cell_type":"code","source":"train['timestamp'].nunique()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:13.794087Z","iopub.execute_input":"2021-12-02T03:08:13.794292Z","iopub.status.idle":"2021-12-02T03:08:14.36469Z","shell.execute_reply.started":"2021-12-02T03:08:13.794267Z","shell.execute_reply":"2021-12-02T03:08:14.363842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### We have total of 1.9 Million timestamps","metadata":{}},{"cell_type":"markdown","source":"### Lets explore assets csv file","metadata":{}},{"cell_type":"code","source":"assets","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:14.365959Z","iopub.execute_input":"2021-12-02T03:08:14.366511Z","iopub.status.idle":"2021-12-02T03:08:14.377657Z","shell.execute_reply.started":"2021-12-02T03:08:14.366459Z","shell.execute_reply":"2021-12-02T03:08:14.376798Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets check how weights are distributed","metadata":{}},{"cell_type":"code","source":"plt.subplots(figsize=(20,10))\nsns.barplot(x='Asset_Name', y='Weight', data=assets.sort_values(by=['Weight'],ascending=False))","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:14.378823Z","iopub.execute_input":"2021-12-02T03:08:14.379045Z","iopub.status.idle":"2021-12-02T03:08:14.755103Z","shell.execute_reply.started":"2021-12-02T03:08:14.379019Z","shell.execute_reply":"2021-12-02T03:08:14.753489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Lets see the data distribution in each timestamp","metadata":{}},{"cell_type":"code","source":"train[train['timestamp'] == 1514764860]","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:14.756125Z","iopub.execute_input":"2021-12-02T03:08:14.756456Z","iopub.status.idle":"2021-12-02T03:08:14.842237Z","shell.execute_reply.started":"2021-12-02T03:08:14.75643Z","shell.execute_reply":"2021-12-02T03:08:14.84151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Please note that there are total of 14 asset data (crypto currencies) given.\nFor a particular timestamp, we need not necessarily have a record of each asset. \nFrom above table, we have only 7 asset details. Target value for one of them is NaN. ","metadata":{}},{"cell_type":"markdown","source":"# Cleaning","metadata":{}},{"cell_type":"markdown","source":"### Lets check missing values","metadata":{}},{"cell_type":"code","source":"train.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:14.844301Z","iopub.execute_input":"2021-12-02T03:08:14.84461Z","iopub.status.idle":"2021-12-02T03:08:15.296965Z","shell.execute_reply.started":"2021-12-02T03:08:14.844574Z","shell.execute_reply":"2021-12-02T03:08:15.2962Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Join Asset tables for Analysis.","metadata":{}},{"cell_type":"code","source":"df_train = pd.merge(train, assets, how=\"left\", on=[\"Asset_ID\"])\ndf_train.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:15.298038Z","iopub.execute_input":"2021-12-02T03:08:15.298317Z","iopub.status.idle":"2021-12-02T03:08:18.821105Z","shell.execute_reply.started":"2021-12-02T03:08:15.298289Z","shell.execute_reply":"2021-12-02T03:08:18.820302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Vizualize","metadata":{}},{"cell_type":"markdown","source":"### Lets see total distribution per bitcoin","metadata":{}},{"cell_type":"code","source":"df = df_train.dropna()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:18.822184Z","iopub.execute_input":"2021-12-02T03:08:18.822399Z","iopub.status.idle":"2021-12-02T03:08:23.947211Z","shell.execute_reply.started":"2021-12-02T03:08:18.822373Z","shell.execute_reply":"2021-12-02T03:08:23.946281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20,10))\nsns.countplot(x='Asset_Name', data = df)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:23.948301Z","iopub.execute_input":"2021-12-02T03:08:23.94895Z","iopub.status.idle":"2021-12-02T03:08:39.257222Z","shell.execute_reply.started":"2021-12-02T03:08:23.948892Z","shell.execute_reply":"2021-12-02T03:08:39.255906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Comparatively there are less records in Maker, Dogecoin, IOTA & Monero","metadata":{}},{"cell_type":"code","source":"df.sample(20)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:39.259002Z","iopub.execute_input":"2021-12-02T03:08:39.259323Z","iopub.status.idle":"2021-12-02T03:08:40.457256Z","shell.execute_reply.started":"2021-12-02T03:08:39.259279Z","shell.execute_reply":"2021-12-02T03:08:40.456483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.subplots(figsize=(20,10))\nsns.heatmap(df.corr())","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:40.458279Z","iopub.execute_input":"2021-12-02T03:08:40.458487Z","iopub.status.idle":"2021-12-02T03:08:51.613384Z","shell.execute_reply.started":"2021-12-02T03:08:40.458462Z","shell.execute_reply":"2021-12-02T03:08:51.612607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Correlation here is Obvious as Open, Close, High & Low, VWAP (avg volume) values should be very close within a timestamp. Please note that no signaificant correlation found for Target column","metadata":{}},{"cell_type":"code","source":"def reduce_mem_usage(df,do_categoricals=False):\n    \"\"\" iterate through all the columns of a dataframe and modify the data type\n        to reduce memory usage.        \n    \"\"\"\n    start_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage of dataframe is {:.2f} MB'.format(start_mem))\n    \n    for col in df.columns:\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            if do_categoricals==True:\n                df[col] = df[col].astype('category')\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print('Memory usage after optimization is: {:.2f} MB'.format(end_mem))\n    print('Decreased by {:.1f}%'.format(100 * (start_mem - end_mem) / start_mem))   \n    return df\n\ndtrain = reduce_mem_usage(df)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:51.614435Z","iopub.execute_input":"2021-12-02T03:08:51.614661Z","iopub.status.idle":"2021-12-02T03:08:56.34587Z","shell.execute_reply.started":"2021-12-02T03:08:51.614634Z","shell.execute_reply":"2021-12-02T03:08:56.345028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtrain['fulldate'] = pd.to_datetime(dtrain['timestamp'], unit='s')\ndtrain['date'] = dtrain['fulldate'].apply(lambda d: d.date())\ndtrain['time'] = dtrain['fulldate'].apply(lambda d: d.time())\ndtrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:08:56.346945Z","iopub.execute_input":"2021-12-02T03:08:56.347172Z","iopub.status.idle":"2021-12-02T03:11:42.066038Z","shell.execute_reply.started":"2021-12-02T03:08:56.347146Z","shell.execute_reply":"2021-12-02T03:11:42.065199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtrain_sample = dtrain.sample(10000)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:11:42.067668Z","iopub.execute_input":"2021-12-02T03:11:42.067967Z","iopub.status.idle":"2021-12-02T03:11:45.704589Z","shell.execute_reply.started":"2021-12-02T03:11:42.06793Z","shell.execute_reply":"2021-12-02T03:11:45.703736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dtrain_sample.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:11:45.705725Z","iopub.execute_input":"2021-12-02T03:11:45.70594Z","iopub.status.idle":"2021-12-02T03:11:45.725356Z","shell.execute_reply.started":"2021-12-02T03:11:45.705914Z","shell.execute_reply":"2021-12-02T03:11:45.724483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, axes = plt.subplots(7, 2, figsize=(20, 50))\nfor i,asset in enumerate(assets['Asset_Name']):\n    df_crypt = dtrain_sample[dtrain_sample['Asset_Name'] == asset]\n    sns.lineplot(x=\"date\", y=\"VWAP\", data=df_crypt, ax=axes[int(i/2),i%2])\n    axes[int(i/2),i%2].set_title(asset)","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:11:45.726476Z","iopub.execute_input":"2021-12-02T03:11:45.726706Z","iopub.status.idle":"2021-12-02T03:12:28.996346Z","shell.execute_reply.started":"2021-12-02T03:11:45.72668Z","shell.execute_reply":"2021-12-02T03:12:28.995628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Available data period\")\nfor i,asset in enumerate(assets['Asset_Name']):\n    df_crypt = dtrain[dtrain['Asset_Name'] == asset]\n    print( \"{} to {} ------> {}\".format(df_crypt.sort_values(by=['timestamp'])['date'].iloc[0], df_crypt.sort_values(by=['timestamp'])['date'].iloc[-1], asset ))","metadata":{"execution":{"iopub.status.busy":"2021-12-02T03:12:28.99742Z","iopub.execute_input":"2021-12-02T03:12:28.998204Z","iopub.status.idle":"2021-12-02T03:13:00.648874Z","shell.execute_reply.started":"2021-12-02T03:12:28.998169Z","shell.execute_reply":"2021-12-02T03:13:00.648007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# References\n\n1. [Let's Talk Validation: GroupTimeSeriesSplit](https://www.kaggle.com/yamqwe/let-s-talk-validation-grouptimeseriessplit/notebook#References)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}