{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-17T22:14:48.093158Z","iopub.execute_input":"2021-11-17T22:14:48.093417Z","iopub.status.idle":"2021-11-17T22:14:48.101391Z","shell.execute_reply.started":"2021-11-17T22:14:48.093390Z","shell.execute_reply":"2021-11-17T22:14:48.100510Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_folder = \"../input/g-research-crypto-forecasting/\"\ncrypto_df = pd.read_csv(data_folder + 'train.csv')\nasset_details_df = pd.read_csv(data_folder + 'asset_details.csv')\nsupp_train_df = pd.read_csv(data_folder + 'supplemental_train.csv')\nexpl_test = pd.read_csv(data_folder + 'example_test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-11-17T22:14:56.368982Z","iopub.execute_input":"2021-11-17T22:14:56.369432Z","iopub.status.idle":"2021-11-17T22:16:06.591514Z","shell.execute_reply.started":"2021-11-17T22:14:56.369400Z","shell.execute_reply":"2021-11-17T22:16:06.590125Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#replace the spaces inside the asset name with undescore '_'\nasset_details_df[\"Asset_Name\"] = asset_details_df[\"Asset_Name\"].str.replace(' ','_')\nasset_details_df[\"Asset_Name\"] = asset_details_df[\"Asset_Name\"].str.replace('.','_')","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:52:33.100307Z","iopub.execute_input":"2021-11-17T21:52:33.100488Z","iopub.status.idle":"2021-11-17T21:52:33.108189Z","shell.execute_reply.started":"2021-11-17T21:52:33.100467Z","shell.execute_reply":"2021-11-17T21:52:33.106987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details = asset_details_df.copy()","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:52:33.109518Z","iopub.execute_input":"2021-11-17T21:52:33.109719Z","iopub.status.idle":"2021-11-17T21:52:33.129774Z","shell.execute_reply.started":"2021-11-17T21:52:33.109695Z","shell.execute_reply":"2021-11-17T21:52:33.128806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Create a dictionary of data frames assigned to each coin \ndataframes = {}\nfor asset_id, asset_name in zip(asset_details_df.Asset_ID, asset_details_df.Asset_Name):    \n    keys = range(4)\n    vars()[asset_name] = crypto_df[crypto_df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n    dataframes[asset_id] = vars()[asset_name]","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:52:33.130910Z","iopub.execute_input":"2021-11-17T21:52:33.131625Z","iopub.status.idle":"2021-11-17T21:52:38.177307Z","shell.execute_reply.started":"2021-11-17T21:52:33.131547Z","shell.execute_reply":"2021-11-17T21:52:38.168444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"asset_details_df = asset_details_df.set_index(\"Asset_ID\")","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:52:38.187111Z","iopub.execute_input":"2021-11-17T21:52:38.190236Z","iopub.status.idle":"2021-11-17T21:52:38.204408Z","shell.execute_reply.started":"2021-11-17T21:52:38.190189Z","shell.execute_reply":"2021-11-17T21:52:38.202128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(dataframes)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T22:31:37.344806Z","iopub.execute_input":"2021-11-17T22:31:37.346108Z","iopub.status.idle":"2021-11-17T22:31:37.438302Z","shell.execute_reply.started":"2021-11-17T22:31:37.346059Z","shell.execute_reply":"2021-11-17T22:31:37.436874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Some of the records are not received and we can check it by calculating the difference between the timestamps\n#In order to resolve this we need to reindex each of our dataset\ncleaned_dataframes = {}\nfor i in list(dataframes):\n    cleaned_dataframes[i] = dataframes[i].reindex(range(dataframes[i].index[0],dataframes[i].index[-1]+60,60),method='pad')","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:53:27.828269Z","iopub.execute_input":"2021-11-17T21:53:27.828804Z","iopub.status.idle":"2021-11-17T21:53:33.954956Z","shell.execute_reply.started":"2021-11-17T21:53:27.828760Z","shell.execute_reply":"2021-11-17T21:53:33.953700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Reduce the datastes to ease the correlation calculation\nfrom datetime import datetime\nimport time\ntotimestamp = lambda s: np.int32(time.mktime(datetime.strptime(s, \"%d/%m/%Y\").timetuple()))\nreduced_dataframes = {}\nfor i in list(cleaned_dataframes):\n    reduced_dataframes[i] = cleaned_dataframes[i].loc[totimestamp('01/01/2021'):totimestamp('01/05/2021')]\n    #reduced_dataframes[i] = cleaned_dataframes[i].iloc[-100:]","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:53:33.957659Z","iopub.execute_input":"2021-11-17T21:53:33.958663Z","iopub.status.idle":"2021-11-17T21:53:33.989599Z","shell.execute_reply.started":"2021-11-17T21:53:33.958596Z","shell.execute_reply":"2021-11-17T21:53:33.987733Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(reduced_dataframes)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T18:54:14.188453Z","iopub.execute_input":"2021-11-17T18:54:14.188673Z","iopub.status.idle":"2021-11-17T18:54:14.206993Z","shell.execute_reply.started":"2021-11-17T18:54:14.188647Z","shell.execute_reply":"2021-11-17T18:54:14.2064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import plotly.graph_objects as go\nfor i in list(reduced_dataframes) :\n    print(i)\n    print(asset_details_df.Asset_Name[i])\n    fig = go.Figure(data=[go.Candlestick(x=reduced_dataframes[i].index, open=reduced_dataframes[i]['Open'], high=reduced_dataframes[i]['High'], low=reduced_dataframes[i]['Low'], close=reduced_dataframes[i]['Close'])])\n    fig.show()","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:48:21.676165Z","iopub.execute_input":"2021-11-17T21:48:21.67647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define function to compute log returns\ndef log_return(series, periods=1):\n    return np.log(series).diff(periods=periods)","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:53:42.951644Z","iopub.execute_input":"2021-11-17T21:53:42.952091Z","iopub.status.idle":"2021-11-17T21:53:42.956705Z","shell.execute_reply.started":"2021-11-17T21:53:42.952064Z","shell.execute_reply":"2021-11-17T21:53:42.955046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_cryptos = pd.DataFrame([])\nfor i in list(reduced_dataframes) :\n    lret = log_return(reduced_dataframes[i].Close.fillna(0))[1:]\n    corr_cryptos = corr_cryptos.join(lret, rsuffix=asset_details_df[\"Asset_Name\"][i], how=\"outer\")","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:53:48.634983Z","iopub.execute_input":"2021-11-17T21:53:48.635500Z","iopub.status.idle":"2021-11-17T21:53:48.779937Z","shell.execute_reply.started":"2021-11-17T21:53:48.635463Z","shell.execute_reply":"2021-11-17T21:53:48.779172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr_cryptos","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:53:52.048807Z","iopub.execute_input":"2021-11-17T21:53:52.049060Z","iopub.status.idle":"2021-11-17T21:53:52.085980Z","shell.execute_reply.started":"2021-11-17T21:53:52.049035Z","shell.execute_reply":"2021-11-17T21:53:52.085562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.imshow(corr_cryptos.corr());\nplt.yticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values);\nplt.xticks(asset_details.Asset_ID.values, asset_details.Asset_Name.values, rotation='vertical');\nplt.colorbar();","metadata":{"execution":{"iopub.status.busy":"2021-11-17T21:54:04.864874Z","iopub.execute_input":"2021-11-17T21:54:04.865127Z","iopub.status.idle":"2021-11-17T21:54:05.350769Z","shell.execute_reply.started":"2021-11-17T21:54:04.865099Z","shell.execute_reply":"2021-11-17T21:54:05.348752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"execution":{"iopub.status.busy":"2021-11-16T17:02:20.825798Z","iopub.execute_input":"2021-11-16T17:02:20.826172Z","iopub.status.idle":"2021-11-16T17:02:20.850653Z","shell.execute_reply.started":"2021-11-16T17:02:20.826134Z","shell.execute_reply":"2021-11-16T17:02:20.849522Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}