{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport pandas as pd\nimport numpy as np\nfrom datetime import datetime\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport plotly.express as px\n\nimport seaborn as sns\ncmap = sns.color_palette()\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-25T18:59:38.659279Z","iopub.execute_input":"2022-01-25T18:59:38.659545Z","iopub.status.idle":"2022-01-25T18:59:38.675153Z","shell.execute_reply.started":"2022-01-25T18:59:38.659515Z","shell.execute_reply":"2022-01-25T18:59:38.674039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"we can see here that weights column in assets_details will be used to get the weighted pearson correlation coefficient evaluation metric.","metadata":{}},{"cell_type":"code","source":"assets = pd.read_csv(\"../input/g-research-crypto-forecasting/asset_details.csv\")\nassets.sort_values(\"Weight\", ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:39.728171Z","iopub.execute_input":"2022-01-25T18:59:39.728896Z","iopub.status.idle":"2022-01-25T18:59:39.750753Z","shell.execute_reply.started":"2022-01-25T18:59:39.728852Z","shell.execute_reply":"2022-01-25T18:59:39.749957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Whole Distribution of weights of different assets into pie chart","metadata":{}},{"cell_type":"code","source":"assets[\"weight_percent\"] = (assets[\"Weight\"] / assets[\"Weight\"].sum()) * 100\nassets.sort_values(\"Weight\", ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:40.628001Z","iopub.execute_input":"2022-01-25T18:59:40.628364Z","iopub.status.idle":"2022-01-25T18:59:40.644315Z","shell.execute_reply.started":"2022-01-25T18:59:40.628313Z","shell.execute_reply":"2022-01-25T18:59:40.643512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Get The Pie chart for above weight_percentage feature","metadata":{}},{"cell_type":"code","source":"fig = px.pie(assets, values='weight_percent', names='Asset_Name', title='Weights assigned to each cryptocurrency')\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:41.337064Z","iopub.execute_input":"2022-01-25T18:59:41.337586Z","iopub.status.idle":"2022-01-25T18:59:41.384553Z","shell.execute_reply.started":"2022-01-25T18:59:41.337548Z","shell.execute_reply":"2022-01-25T18:59:41.383733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"assets.columns","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:41.685748Z","iopub.execute_input":"2022-01-25T18:59:41.686261Z","iopub.status.idle":"2022-01-25T18:59:41.691472Z","shell.execute_reply.started":"2022-01-25T18:59:41.686224Z","shell.execute_reply":"2022-01-25T18:59:41.690546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names_dict = {row[\"Asset_Name\"]:row[\"Asset_ID\"] for ind, row in assets.iterrows()}","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:42.023462Z","iopub.execute_input":"2022-01-25T18:59:42.024015Z","iopub.status.idle":"2022-01-25T18:59:42.029756Z","shell.execute_reply.started":"2022-01-25T18:59:42.023974Z","shell.execute_reply":"2022-01-25T18:59:42.028820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names_dict","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:42.402574Z","iopub.execute_input":"2022-01-25T18:59:42.403123Z","iopub.status.idle":"2022-01-25T18:59:42.408378Z","shell.execute_reply.started":"2022-01-25T18:59:42.403084Z","shell.execute_reply":"2022-01-25T18:59:42.407724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv(\"../input/g-research-crypto-forecasting/train.csv\")\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-25T18:59:49.760060Z","iopub.execute_input":"2022-01-25T18:59:49.760699Z","iopub.status.idle":"2022-01-25T19:00:21.504831Z","shell.execute_reply.started":"2022-01-25T18:59:49.760636Z","shell.execute_reply":"2022-01-25T19:00:21.504038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_names = [\n    'Bitcoin',\n    'Ethereum',\n    'Cardano',\n    'Binance Coin',\n    'Dogecoin',\n    'Bitcoin Cash',\n    'Litecoin',\n    'Ethereum Classic',\n    'Stellar',\n    'TRON',\n    'Monero',\n    'EOS.IO',\n    'IOTA',\n    'Maker'\n]\n\ntime_frame = []\nfor asset in asset_names:\n    #fetching the asset whose id is similar to any particular asset in that dataframe and gather into one dataframe\n#     print(coin_df)\n    coin_df = train[train[\"Asset_ID\"]==asset_names_dict[asset]].set_index(\"timestamp\")\n    start_time = coin_df.index[0].astype('datetime64[s]')\n#     print(start_time)\n    end_time = coin_df.index[-1].astype('datetime64[s]')\n#     print(end_time)\n    time_frame.append([asset, start_time, end_time])\ntime_df = pd.DataFrame(time_frame)\ntime_df.columns = [\"Asset_Name\", \"Start_Time\", \"End_Time\"]\ntime_df","metadata":{"execution":{"iopub.status.busy":"2022-01-25T19:00:36.557991Z","iopub.execute_input":"2022-01-25T19:00:36.558236Z","iopub.status.idle":"2022-01-25T19:00:40.438099Z","shell.execute_reply.started":"2022-01-25T19:00:36.558208Z","shell.execute_reply":"2022-01-25T19:00:40.437423Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Inference from above Observation:\n\n* The earliest start date in the data is Jan 1, 2018 and most coins have that start datetime.\n* The data is available till Sep 21, 2021 and all the coins have the same end datetime.\n* Dogecoin has the least historical information available of the given coins in the given data and it is captured only from April 2019 instead of others assets.","metadata":{}},{"cell_type":"markdown","source":"Missing data\n","metadata":{}},{"cell_type":"code","source":"missing_list = []\nfor asset in asset_names:\n    coin_df =train[train[\"Asset_ID\"]==asset_names_dict[asset]].set_index(\"timestamp\")\n    missing_list.append([asset, coin_df.shape[0]] + coin_df.isna().sum().tolist())\nmissing_df = pd.DataFrame(missing_list)\nmissing_df.columns = [\"Asset_Name\", \"TotalRows\", 'Missing_Asset_ID', 'Missing_Count', 'Missing_Open', \n                      'Missing_High', 'Missing_Low', 'Missing_Close', 'Missing_Volume', 'Missing_VWAP', 'Missing_Target']\nmissing_df","metadata":{"execution":{"iopub.status.busy":"2022-01-25T19:00:40.439702Z","iopub.execute_input":"2022-01-25T19:00:40.439959Z","iopub.status.idle":"2022-01-25T19:00:44.641190Z","shell.execute_reply.started":"2022-01-25T19:00:40.439926Z","shell.execute_reply":"2022-01-25T19:00:44.640521Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Inference from above missing data Information:\n\n* There is a lot of missing data in the Target column\n* All the other columns do not have missing data\n* Bitcoin has the least number of missing target values and Monero the most.\n","metadata":{}},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()\niter_test = env.iter_test()\n\ntrain_df = pd.read_csv('/kaggle/input/g-research-crypto-forecasting/train.csv',usecols=['Target', 'Asset_ID','timestamp'], dtype={'Asset_ID': 'int8'})\ntrain_df['datetime'] = pd.to_datetime(train_df['timestamp'], unit='s')\ntrain_df = train_df.set_index('datetime').drop('timestamp', axis=1)\ntrain_df = train_df[(train_df.index.year == 2021) & (train_df.index.month > 5)]\ndfs = {asset_id: train_df[train_df['Asset_ID'] == asset_id].resample('1min').interpolate().copy() for asset_id in train_df['Asset_ID'].unique()}\ndel train_df\nfor df_test, df_pred in iter_test:\n    df_test['datetime'] = pd.to_datetime(df_test['timestamp'], unit='s')\n    for _, row in df_test.iterrows():\n        try:\n            df = dfs[row['Asset_ID']]\n            closest_train_sample = df.iloc[df.index.get_loc(row['datetime'], method='nearest')]\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = closest_train_sample['Target']\n        except:\n            df_pred.loc[df_pred['row_id'] == row['row_id'], 'Target'] = 0\n    df_pred['Target'] = df_pred['Target'].fillna(0)\n    env.predict(df_pred)","metadata":{"execution":{"iopub.status.busy":"2022-01-25T19:03:59.895820Z","iopub.execute_input":"2022-01-25T19:03:59.896271Z","iopub.status.idle":"2022-01-25T19:04:28.476723Z","shell.execute_reply.started":"2022-01-25T19:03:59.896231Z","shell.execute_reply":"2022-01-25T19:04:28.475806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}