{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot  as plt\nimport gc\nimport lightgbm as lgb\n# from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.linear_model import Ridge\nfrom joblib import Parallel, delayed\nimport gc\nimport seaborn as sns\nfrom scipy import stats\nimport time \nfrom joblib import Parallel, delayed\nfrom sklearn.manifold import TSNE\nfrom sklearn.preprocessing import minmax_scale\nfrom sklearn.decomposition import PCA\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA\nfrom yellowbrick.cluster import KElbowVisualizer\nfrom sklearn.cluster import KMeans, DBSCAN\nfrom sklearn.neighbors import NearestNeighbors,KNeighborsClassifier, NeighborhoodComponentsAnalysis, KNeighborsRegressor, LocalOutlierFactor\nfrom sklearn.cluster import AgglomerativeClustering\n\nimport plotly.io as plt_io\nimport plotly.graph_objects as go\n\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-23T08:27:47.706246Z","iopub.execute_input":"2021-11-23T08:27:47.706928Z","iopub.status.idle":"2021-11-23T08:27:48.047961Z","shell.execute_reply.started":"2021-11-23T08:27:47.706871Z","shell.execute_reply":"2021-11-23T08:27:48.046966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ndetails = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\ndetails.sort_values(by=['Asset_ID'], inplace = True)\nprint(details)\n\ndef read_csv_strict(file_name='/kaggle/input/g-research-crypto-forecasting/train.csv'):\n    df = pd.read_csv(file_name)\n    \n    df2 = pd.read_csv('../input/g-research-crypto-forecasting/supplemental_train.csv')\n\n    df = pd.concat([df,df2])\n    \n    df.sort_values(['Asset_ID','timestamp'],ascending=True, inplace=True)\n    df.reset_index(drop=True, inplace=True)\n    df['Weight'] = df.Asset_ID.map(details.Weight)\n    \n    \n    df['VWAP'] = np.where(np.isinf(df.VWAP),df.Close,df.VWAP)\n\n        \n    df['date_time'] = pd.to_datetime(df['timestamp'], unit='s')\n    df_tr = df[df['date_time'] > '2021-01-01 00:00:00']\n    df_ts = df[df['date_time'] >= '2021-06-13 00:00:00']\n        \n    return df_tr, df_ts\ndf, df_ts = read_csv_strict()\n\ndel read_csv_strict\ndf.tail(20)","metadata":{"execution":{"iopub.status.busy":"2021-11-23T08:21:03.695152Z","iopub.execute_input":"2021-11-23T08:21:03.695622Z","iopub.status.idle":"2021-11-23T08:22:27.413133Z","shell.execute_reply.started":"2021-11-23T08:21:03.695588Z","shell.execute_reply":"2021-11-23T08:22:27.412161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dropna(inplace=True)\ndf.reset_index(inplace = True, drop = True)\ndf","metadata":{"execution":{"iopub.status.busy":"2021-11-23T08:26:08.906108Z","iopub.execute_input":"2021-11-23T08:26:08.90645Z","iopub.status.idle":"2021-11-23T08:26:11.595588Z","shell.execute_reply.started":"2021-11-23T08:26:08.906415Z","shell.execute_reply":"2021-11-23T08:26:11.594547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_idx_timestamp = df.pivot(index='timestamp', columns='Asset_ID', values='Target')\ndf_idx_Asset_ID = df.pivot(index='Asset_ID', columns='timestamp', values='Target')\ndf_idx_timestamp","metadata":{"execution":{"iopub.status.busy":"2021-11-23T08:26:36.764567Z","iopub.execute_input":"2021-11-23T08:26:36.764856Z","iopub.status.idle":"2021-11-23T08:27:05.513366Z","shell.execute_reply.started":"2021-11-23T08:26:36.764824Z","shell.execute_reply":"2021-11-23T08:27:05.512275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_idx_timestamp.dropna(inplace=True)\ndf_idx_timestamp.reset_index(inplace = True, drop = True)\ndf_idx_Asset_ID = df_idx_timestamp.T","metadata":{"execution":{"iopub.status.busy":"2021-11-23T08:29:41.602174Z","iopub.execute_input":"2021-11-23T08:29:41.602929Z","iopub.status.idle":"2021-11-23T08:29:41.651104Z","shell.execute_reply.started":"2021-11-23T08:29:41.602885Z","shell.execute_reply":"2021-11-23T08:29:41.649721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"neigh = NearestNeighbors(n_neighbors=6)\nnbrs = neigh.fit(df_idx_Asset_ID)\nneighb = nbrs.kneighbors(df_idx_Asset_ID,  return_distance=False)\n\nneighb\nfor i,x in enumerate(df_idx_Asset_ID.index):\n    print(neighb[i][0], neighb[i][1], neighb[i][2],neighb[i][3],neighb[i][4],neighb[i][5])","metadata":{"execution":{"iopub.status.busy":"2021-11-23T08:31:49.494751Z","iopub.execute_input":"2021-11-23T08:31:49.495307Z","iopub.status.idle":"2021-11-23T08:31:55.906545Z","shell.execute_reply.started":"2021-11-23T08:31:49.495254Z","shell.execute_reply":"2021-11-23T08:31:55.905288Z"},"trusted":true},"execution_count":null,"outputs":[]}]}