{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib.pyplot as plt\nimport seaborn as sns","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-22T06:50:19.896392Z","iopub.execute_input":"2022-05-22T06:50:19.89678Z","iopub.status.idle":"2022-05-22T06:50:20.061614Z","shell.execute_reply.started":"2022-05-22T06:50:19.896719Z","shell.execute_reply":"2022-05-22T06:50:20.060661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T06:50:20.425913Z","iopub.execute_input":"2022-05-22T06:50:20.426281Z","iopub.status.idle":"2022-05-22T06:51:24.619408Z","shell.execute_reply.started":"2022-05-22T06:50:20.426238Z","shell.execute_reply":"2022-05-22T06:51:24.618329Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T06:51:24.62133Z","iopub.execute_input":"2022-05-22T06:51:24.621619Z","iopub.status.idle":"2022-05-22T06:51:36.533001Z","shell.execute_reply.started":"2022-05-22T06:51:24.621585Z","shell.execute_reply":"2022-05-22T06:51:36.532067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# plot close values as time series for all the assets\nf = plt.figure(figsize=(15,30))\n\nfor asset_id in range(10):\n    coin_df = df[df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")\n    ax = f.add_subplot(7,2,asset_id+1)\n    plt.plot(coin_df['Close'], label=asset_id)\n    plt.legend()\n    plt.xlabel('Time')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-05-22T06:51:36.534391Z","iopub.execute_input":"2022-05-22T06:51:36.534662Z","iopub.status.idle":"2022-05-22T06:51:53.134982Z","shell.execute_reply.started":"2022-05-22T06:51:36.53463Z","shell.execute_reply":"2022-05-22T06:51:53.12336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coins_df = pd.DataFrame()\n\nfor asset_id in range(10):\n    coin_df = df[df[\"Asset_ID\"]==asset_id].set_index(\"timestamp\")[['Close']].rename(columns={'Close':asset_id})\n    coins_df = pd.concat([coins_df,coin_df], axis=1)\n\ncoins_df = coins_df.pct_change() #get returns\ncoins_df = (coins_df-coins_df.min())/ (coins_df.max() - coins_df.min())\ncoins_df.dropna(inplace=True) #just ignoring the liberties I'm taking here","metadata":{"execution":{"iopub.status.busy":"2022-05-22T06:51:53.137469Z","iopub.execute_input":"2022-05-22T06:51:53.138324Z","iopub.status.idle":"2022-05-22T06:52:05.165052Z","shell.execute_reply.started":"2022-05-22T06:51:53.138266Z","shell.execute_reply":"2022-05-22T06:52:05.163789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# induce a sudden spike anomaly\ncoins_df.loc[1555080360,2] = 0.7","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:18:34.23961Z","iopub.execute_input":"2022-05-22T07:18:34.239942Z","iopub.status.idle":"2022-05-22T07:18:34.245471Z","shell.execute_reply.started":"2022-05-22T07:18:34.239909Z","shell.execute_reply":"2022-05-22T07:18:34.244363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get a sense of how noisy this data is by colour coding the values\ncm = sns.light_palette(\"green\", as_cmap=True)\ncoins_df.head(20).style.background_gradient(cmap=cm,axis=None).set_precision(4)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:18:35.07658Z","iopub.execute_input":"2022-05-22T07:18:35.077202Z","iopub.status.idle":"2022-05-22T07:18:35.121432Z","shell.execute_reply.started":"2022-05-22T07:18:35.077157Z","shell.execute_reply":"2022-05-22T07:18:35.120252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"coins_df.plot(figsize=(15,5), alpha=0.5);","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:18:36.016689Z","iopub.execute_input":"2022-05-22T07:18:36.01701Z","iopub.status.idle":"2022-05-22T07:19:16.627953Z","shell.execute_reply.started":"2022-05-22T07:18:36.016977Z","shell.execute_reply":"2022-05-22T07:19:16.626895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.decomposition import PCA\nfrom numpy.testing import assert_array_almost_equal\n\n# too much memory for the large dataset\nX_train = coins_df.head(10000)\n\npca = PCA(n_components=3)\npca.fit(X_train)\n\nX_train_pca = pca.transform(X_train)\n\nX_projected = pca.inverse_transform(X_train_pca)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:19:16.630559Z","iopub.execute_input":"2022-05-22T07:19:16.630934Z","iopub.status.idle":"2022-05-22T07:19:16.691783Z","shell.execute_reply.started":"2022-05-22T07:19:16.630892Z","shell.execute_reply":"2022-05-22T07:19:16.690665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_projected = pd.DataFrame(X_projected, index=coins_df.index[:len(X_projected)])\nX_projected.head(20).style.background_gradient(cmap=cm,axis=None).set_precision(4)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:19:16.698689Z","iopub.execute_input":"2022-05-22T07:19:16.702833Z","iopub.status.idle":"2022-05-22T07:19:16.817235Z","shell.execute_reply.started":"2022-05-22T07:19:16.702752Z","shell.execute_reply":"2022-05-22T07:19:16.816238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train - X_projected).head(20).style.background_gradient(cmap=cm,axis=None).set_precision(4)","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:19:16.819794Z","iopub.execute_input":"2022-05-22T07:19:16.820169Z","iopub.status.idle":"2022-05-22T07:19:16.861417Z","shell.execute_reply.started":"2022-05-22T07:19:16.820121Z","shell.execute_reply":"2022-05-22T07:19:16.860162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train).head(20).plot();","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:19:16.863026Z","iopub.execute_input":"2022-05-22T07:19:16.863341Z","iopub.status.idle":"2022-05-22T07:19:17.127672Z","shell.execute_reply.started":"2022-05-22T07:19:16.863282Z","shell.execute_reply":"2022-05-22T07:19:17.126571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(X_train - X_projected).head(20).plot();","metadata":{"execution":{"iopub.status.busy":"2022-05-22T07:19:17.129269Z","iopub.execute_input":"2022-05-22T07:19:17.129645Z","iopub.status.idle":"2022-05-22T07:19:17.365359Z","shell.execute_reply.started":"2022-05-22T07:19:17.129588Z","shell.execute_reply":"2022-05-22T07:19:17.364371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}