{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-11-05T11:26:09.867761Z","iopub.execute_input":"2021-11-05T11:26:09.868077Z","iopub.status.idle":"2021-11-05T11:26:09.900759Z","shell.execute_reply.started":"2021-11-05T11:26:09.867992Z","shell.execute_reply":"2021-11-05T11:26:09.899797Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Importing Libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.express as px\nfrom sklearn import preprocessing \nfrom category_encoders import *\nfrom sklearn.preprocessing import LabelEncoder\n%matplotlib inline\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LinearRegression\nfrom sklearn.metrics import r2_score,mean_squared_error\nfrom sklearn import datasets, linear_model, metrics\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import preprocessing \nfrom sklearn.impute import SimpleImputer","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:26:09.903072Z","iopub.execute_input":"2021-11-05T11:26:09.903781Z","iopub.status.idle":"2021-11-05T11:26:13.787216Z","shell.execute_reply.started":"2021-11-05T11:26:09.903739Z","shell.execute_reply":"2021-11-05T11:26:13.786279Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Loading asset details dataset","metadata":{}},{"cell_type":"code","source":"df_asset = pd.read_csv('../input/g-research-crypto-forecasting/asset_details.csv')\ndf_asset","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:26:13.790255Z","iopub.execute_input":"2021-11-05T11:26:13.790913Z","iopub.status.idle":"2021-11-05T11:26:13.821731Z","shell.execute_reply.started":"2021-11-05T11:26:13.790852Z","shell.execute_reply":"2021-11-05T11:26:13.820638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Loading Train Data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv('../input/g-research-crypto-forecasting/train.csv')\ndf","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:26:13.823461Z","iopub.execute_input":"2021-11-05T11:26:13.823794Z","iopub.status.idle":"2021-11-05T11:27:12.240832Z","shell.execute_reply.started":"2021-11-05T11:26:13.823753Z","shell.execute_reply":"2021-11-05T11:27:12.239829Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Exploratory Data Analysis of train data","metadata":{}},{"cell_type":"code","source":"df.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.243855Z","iopub.execute_input":"2021-11-05T11:27:12.244146Z","iopub.status.idle":"2021-11-05T11:27:12.261683Z","shell.execute_reply.started":"2021-11-05T11:27:12.244103Z","shell.execute_reply":"2021-11-05T11:27:12.260583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.tail()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.263347Z","iopub.execute_input":"2021-11-05T11:27:12.267470Z","iopub.status.idle":"2021-11-05T11:27:12.302606Z","shell.execute_reply.started":"2021-11-05T11:27:12.267418Z","shell.execute_reply":"2021-11-05T11:27:12.301481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.304000Z","iopub.execute_input":"2021-11-05T11:27:12.304276Z","iopub.status.idle":"2021-11-05T11:27:12.311876Z","shell.execute_reply.started":"2021-11-05T11:27:12.304245Z","shell.execute_reply":"2021-11-05T11:27:12.310590Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.size","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.314262Z","iopub.execute_input":"2021-11-05T11:27:12.315332Z","iopub.status.idle":"2021-11-05T11:27:12.325254Z","shell.execute_reply.started":"2021-11-05T11:27:12.315260Z","shell.execute_reply":"2021-11-05T11:27:12.324029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.327350Z","iopub.execute_input":"2021-11-05T11:27:12.328098Z","iopub.status.idle":"2021-11-05T11:27:12.339058Z","shell.execute_reply.started":"2021-11-05T11:27:12.328036Z","shell.execute_reply":"2021-11-05T11:27:12.337772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.341163Z","iopub.execute_input":"2021-11-05T11:27:12.341616Z","iopub.status.idle":"2021-11-05T11:27:12.350075Z","shell.execute_reply.started":"2021-11-05T11:27:12.341486Z","shell.execute_reply":"2021-11-05T11:27:12.348894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.355263Z","iopub.execute_input":"2021-11-05T11:27:12.355997Z","iopub.status.idle":"2021-11-05T11:27:12.379353Z","shell.execute_reply.started":"2021-11-05T11:27:12.355957Z","shell.execute_reply":"2021-11-05T11:27:12.378181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:12.381738Z","iopub.execute_input":"2021-11-05T11:27:12.382962Z","iopub.status.idle":"2021-11-05T11:27:20.603104Z","shell.execute_reply.started":"2021-11-05T11:27:12.382913Z","shell.execute_reply":"2021-11-05T11:27:20.601996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:20.605001Z","iopub.execute_input":"2021-11-05T11:27:20.605625Z","iopub.status.idle":"2021-11-05T11:27:21.169423Z","shell.execute_reply.started":"2021-11-05T11:27:20.605579Z","shell.execute_reply":"2021-11-05T11:27:21.168398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:21.171280Z","iopub.execute_input":"2021-11-05T11:27:21.171639Z","iopub.status.idle":"2021-11-05T11:27:21.176319Z","shell.execute_reply.started":"2021-11-05T11:27:21.171595Z","shell.execute_reply":"2021-11-05T11:27:21.175106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.skew()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:21.177987Z","iopub.execute_input":"2021-11-05T11:27:21.178625Z","iopub.status.idle":"2021-11-05T11:27:26.583593Z","shell.execute_reply.started":"2021-11-05T11:27:21.178565Z","shell.execute_reply":"2021-11-05T11:27:26.582593Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.corr()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:26.585384Z","iopub.execute_input":"2021-11-05T11:27:26.585979Z","iopub.status.idle":"2021-11-05T11:27:34.539384Z","shell.execute_reply.started":"2021-11-05T11:27:26.585929Z","shell.execute_reply":"2021-11-05T11:27:34.538214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Column with null values and their count\nc = 0\nis_null = []\nfor i in df.columns:\n if df[i].isnull().sum()>0:\n is_null.append(i)\n print(i,df[i].isnull().sum())\n c = c+1\nprint('Number of columns containing null values are:',c)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:34.540966Z","iopub.execute_input":"2021-11-05T11:27:34.541374Z","iopub.status.idle":"2021-11-05T11:27:35.113617Z","shell.execute_reply.started":"2021-11-05T11:27:34.541327Z","shell.execute_reply":"2021-11-05T11:27:35.112692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Count of outliers in each numerical columns","metadata":{}},{"cell_type":"code","source":"def count_outliers(data,col):\n q1 = data[col].quantile(0.25,interpolation='nearest')\n q2 = data[col].quantile(0.5,interpolation='nearest')\n q3 = data[col].quantile(0.75,interpolation='nearest')\n q4 = data[col].quantile(1,interpolation='nearest')\n IQR = q3 -q1\n global LLP\n global ULP\n LLP = q1 - 1.5*IQR\n ULP = q3 + 1.5*IQR\n if data[col].min() > LLP and data[col].max() < ULP:\n print(\"No outliers in\",i)\n else:\n print(\"There are outliers in\",i)\n x = data[data[col]ULP][col].size\n a.append(i)\n print('Count of outliers are:',x+y)\nglobal a\na = []\nfor i in df.columns:\n count_outliers(df,i)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:35.115675Z","iopub.execute_input":"2021-11-05T11:27:35.116121Z","iopub.status.idle":"2021-11-05T11:27:53.747366Z","shell.execute_reply.started":"2021-11-05T11:27:35.116079Z","shell.execute_reply":"2021-11-05T11:27:53.746405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Data Preprocessing","metadata":{}},{"cell_type":"code","source":"df.dropna(axis = 0, inplace = True)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:53.752543Z","iopub.execute_input":"2021-11-05T11:27:53.753551Z","iopub.status.idle":"2021-11-05T11:27:56.106987Z","shell.execute_reply.started":"2021-11-05T11:27:53.753484Z","shell.execute_reply":"2021-11-05T11:27:56.106072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.108619Z","iopub.execute_input":"2021-11-05T11:27:56.109184Z","iopub.status.idle":"2021-11-05T11:27:56.603429Z","shell.execute_reply.started":"2021-11-05T11:27:56.109136Z","shell.execute_reply":"2021-11-05T11:27:56.602515Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Cleaning and preprocessing of Test dataset","metadata":{}},{"cell_type":"code","source":"df2 = pd.read_csv('../input/g-research-crypto-forecasting/example_test.csv')\ndf2","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.604743Z","iopub.execute_input":"2021-11-05T11:27:56.605030Z","iopub.status.idle":"2021-11-05T11:27:56.658209Z","shell.execute_reply.started":"2021-11-05T11:27:56.604987Z","shell.execute_reply":"2021-11-05T11:27:56.657367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# dropping last two columns since they are not useful\ndf2.drop(['group_num','row_id'],axis = 1,inplace = True)\ndf2","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.659702Z","iopub.execute_input":"2021-11-05T11:27:56.660015Z","iopub.status.idle":"2021-11-05T11:27:56.700117Z","shell.execute_reply.started":"2021-11-05T11:27:56.659973Z","shell.execute_reply":"2021-11-05T11:27:56.698816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Exploratory Data Analysis of Test data","metadata":{}},{"cell_type":"code","source":"df2.head()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.702125Z","iopub.execute_input":"2021-11-05T11:27:56.702698Z","iopub.status.idle":"2021-11-05T11:27:56.727431Z","shell.execute_reply.started":"2021-11-05T11:27:56.702655Z","shell.execute_reply":"2021-11-05T11:27:56.726544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.tail()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.730313Z","iopub.execute_input":"2021-11-05T11:27:56.730765Z","iopub.status.idle":"2021-11-05T11:27:56.750448Z","shell.execute_reply.started":"2021-11-05T11:27:56.730721Z","shell.execute_reply":"2021-11-05T11:27:56.749021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.shape","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.752346Z","iopub.execute_input":"2021-11-05T11:27:56.753811Z","iopub.status.idle":"2021-11-05T11:27:56.761373Z","shell.execute_reply.started":"2021-11-05T11:27:56.753764Z","shell.execute_reply":"2021-11-05T11:27:56.759945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.size","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.763208Z","iopub.execute_input":"2021-11-05T11:27:56.764621Z","iopub.status.idle":"2021-11-05T11:27:56.772507Z","shell.execute_reply.started":"2021-11-05T11:27:56.764578Z","shell.execute_reply":"2021-11-05T11:27:56.771232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.dtypes","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.774496Z","iopub.execute_input":"2021-11-05T11:27:56.775065Z","iopub.status.idle":"2021-11-05T11:27:56.786829Z","shell.execute_reply.started":"2021-11-05T11:27:56.775014Z","shell.execute_reply":"2021-11-05T11:27:56.785720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.columns","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.788676Z","iopub.execute_input":"2021-11-05T11:27:56.789247Z","iopub.status.idle":"2021-11-05T11:27:56.800513Z","shell.execute_reply.started":"2021-11-05T11:27:56.789200Z","shell.execute_reply":"2021-11-05T11:27:56.799429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.info()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.808102Z","iopub.execute_input":"2021-11-05T11:27:56.808351Z","iopub.status.idle":"2021-11-05T11:27:56.824444Z","shell.execute_reply.started":"2021-11-05T11:27:56.808321Z","shell.execute_reply":"2021-11-05T11:27:56.823008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.describe()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.826120Z","iopub.execute_input":"2021-11-05T11:27:56.827404Z","iopub.status.idle":"2021-11-05T11:27:56.872204Z","shell.execute_reply.started":"2021-11-05T11:27:56.827340Z","shell.execute_reply":"2021-11-05T11:27:56.871215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.873590Z","iopub.execute_input":"2021-11-05T11:27:56.874466Z","iopub.status.idle":"2021-11-05T11:27:56.885358Z","shell.execute_reply.started":"2021-11-05T11:27:56.874389Z","shell.execute_reply":"2021-11-05T11:27:56.883914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.duplicated().sum()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.887100Z","iopub.execute_input":"2021-11-05T11:27:56.888046Z","iopub.status.idle":"2021-11-05T11:27:56.902949Z","shell.execute_reply.started":"2021-11-05T11:27:56.887998Z","shell.execute_reply":"2021-11-05T11:27:56.901959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.skew()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.904777Z","iopub.execute_input":"2021-11-05T11:27:56.905464Z","iopub.status.idle":"2021-11-05T11:27:56.917565Z","shell.execute_reply.started":"2021-11-05T11:27:56.905420Z","shell.execute_reply":"2021-11-05T11:27:56.916130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df2.corr()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.919105Z","iopub.execute_input":"2021-11-05T11:27:56.919448Z","iopub.status.idle":"2021-11-05T11:27:56.943298Z","shell.execute_reply.started":"2021-11-05T11:27:56.919404Z","shell.execute_reply":"2021-11-05T11:27:56.942212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Count of outliers in test data","metadata":{}},{"cell_type":"code","source":"def count_outliers(data,col):\n q1 = data[col].quantile(0.25,interpolation='nearest')\n q2 = data[col].quantile(0.5,interpolation='nearest')\n q3 = data[col].quantile(0.75,interpolation='nearest')\n q4 = data[col].quantile(1,interpolation='nearest')\n IQR = q3 -q1\n global LLP\n global ULP\n LLP = q1 - 1.5*IQR\n ULP = q3 + 1.5*IQR\n if data[col].min() > LLP and data[col].max() < ULP:\n print(\"No outliers in\",i)\n else:\n print(\"There are outliers in\",i)\n x = data[data[col]ULP][col].size\n k.append(i)\n print('Count of outliers are:',x+y)\nglobal k\nk = []\nfor i in df2.columns:\n count_outliers(df2,i)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:27:56.944801Z","iopub.execute_input":"2021-11-05T11:27:56.945727Z","iopub.status.idle":"2021-11-05T11:27:57.012806Z","shell.execute_reply.started":"2021-11-05T11:27:56.945676Z","shell.execute_reply":"2021-11-05T11:27:57.011817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Data Visualization","metadata":{}},{"cell_type":"markdown","source":"> ## Data prep","metadata":{}},{"cell_type":"code","source":"!pip install dataprep","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-05T11:27:57.014053Z","iopub.execute_input":"2021-11-05T11:27:57.014335Z","iopub.status.idle":"2021-11-05T11:28:22.658608Z","shell.execute_reply.started":"2021-11-05T11:27:57.014294Z","shell.execute_reply":"2021-11-05T11:28:22.657091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! python -m pip install \"dask[dataframe]\" --upgrade # or python -m pip install","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-05T11:28:22.660653Z","iopub.execute_input":"2021-11-05T11:28:22.661013Z","iopub.status.idle":"2021-11-05T11:28:34.572539Z","shell.execute_reply.started":"2021-11-05T11:28:22.660959Z","shell.execute_reply":"2021-11-05T11:28:34.571393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from dataprep.eda import plot, plot_correlation, create_report, plot_missing","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:28:34.575055Z","iopub.execute_input":"2021-11-05T11:28:34.575438Z","iopub.status.idle":"2021-11-05T11:28:36.147571Z","shell.execute_reply.started":"2021-11-05T11:28:34.575375Z","shell.execute_reply":"2021-11-05T11:28:36.146461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Visualization for train data","metadata":{}},{"cell_type":"code","source":"plot(df_asset)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:28:36.148988Z","iopub.execute_input":"2021-11-05T11:28:36.150831Z","iopub.status.idle":"2021-11-05T11:28:36.804787Z","shell.execute_reply.started":"2021-11-05T11:28:36.150784Z","shell.execute_reply":"2021-11-05T11:28:36.803564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_report(df_asset)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:28:36.806413Z","iopub.execute_input":"2021-11-05T11:28:36.806937Z","iopub.status.idle":"2021-11-05T11:28:40.006166Z","shell.execute_reply.started":"2021-11-05T11:28:36.806879Z","shell.execute_reply":"2021-11-05T11:28:40.004843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot(df2)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:28:40.008134Z","iopub.execute_input":"2021-11-05T11:28:40.008837Z","iopub.status.idle":"2021-11-05T11:28:41.613087Z","shell.execute_reply.started":"2021-11-05T11:28:40.008757Z","shell.execute_reply":"2021-11-05T11:28:41.611580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"create_report(df2)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:28:41.615690Z","iopub.execute_input":"2021-11-05T11:28:41.616049Z","iopub.status.idle":"2021-11-05T11:28:49.594014Z","shell.execute_reply.started":"2021-11-05T11:28:41.616005Z","shell.execute_reply":"2021-11-05T11:28:49.592583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Visualization with Autoviz","metadata":{}},{"cell_type":"code","source":"! pip install Autoviz","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-05T11:28:49.596587Z","iopub.execute_input":"2021-11-05T11:28:49.597292Z","iopub.status.idle":"2021-11-05T11:29:02.470972Z","shell.execute_reply.started":"2021-11-05T11:28:49.597216Z","shell.execute_reply":"2021-11-05T11:29:02.469551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install xlrd","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-11-05T11:29:02.473767Z","iopub.execute_input":"2021-11-05T11:29:02.474332Z","iopub.status.idle":"2021-11-05T11:29:11.967327Z","shell.execute_reply.started":"2021-11-05T11:29:02.474282Z","shell.execute_reply":"2021-11-05T11:29:11.966037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from autoviz.AutoViz_Class import AutoViz_Class\n\nAV = AutoViz_Class()\ndftc = AV.AutoViz(\n filename='', \n sep='' , \n depVar='Asset_ID', \n dfte=df, \n header=0, \n verbose=1, \n lowess=False, \n chart_format='png', \n max_rows_analyzed=300000, \n max_cols_analyzed=30\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:29:11.970325Z","iopub.execute_input":"2021-11-05T11:29:11.970794Z","iopub.status.idle":"2021-11-05T11:37:06.657805Z","shell.execute_reply.started":"2021-11-05T11:29:11.970726Z","shell.execute_reply":"2021-11-05T11:37:06.656760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from autoviz.AutoViz_Class import AutoViz_Class\n\nAV = AutoViz_Class()\ndftc = AV.AutoViz(\n filename='', \n sep='' , \n depVar='Asset_ID', \n dfte=df2, \n header=0, \n verbose=1, \n lowess=False, \n chart_format='png', \n max_rows_analyzed=300000, \n max_cols_analyzed=30\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:37:06.659886Z","iopub.execute_input":"2021-11-05T11:37:06.660645Z","iopub.status.idle":"2021-11-05T11:37:41.180782Z","shell.execute_reply.started":"2021-11-05T11:37:06.660597Z","shell.execute_reply":"2021-11-05T11:37:41.179585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from autoviz.AutoViz_Class import AutoViz_Class\n\nAV = AutoViz_Class()\ndftc = AV.AutoViz(\n filename='', \n sep='' , \n depVar='Asset_Name', \n dfte=df_asset, \n header=0, \n verbose=1, \n lowess=False, \n chart_format='png', \n max_rows_analyzed=300000, \n max_cols_analyzed=30\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:37:41.182610Z","iopub.execute_input":"2021-11-05T11:37:41.183158Z","iopub.status.idle":"2021-11-05T11:37:45.227618Z","shell.execute_reply.started":"2021-11-05T11:37:41.183109Z","shell.execute_reply":"2021-11-05T11:37:45.226379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from autoviz.AutoViz_Class import AutoViz_Class\n\nAV = AutoViz_Class()\ndftc = AV.AutoViz(\n filename='', \n sep='' , \n depVar='Asset_ID', \n dfte=df_asset, \n header=0, \n verbose=1, \n lowess=False, \n chart_format='png', \n max_rows_analyzed=300000, \n max_cols_analyzed=30\n)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:37:45.230018Z","iopub.execute_input":"2021-11-05T11:37:45.230744Z","iopub.status.idle":"2021-11-05T11:37:48.411738Z","shell.execute_reply.started":"2021-11-05T11:37:45.230680Z","shell.execute_reply":"2021-11-05T11:37:48.410565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import gresearch_crypto\nenv = gresearch_crypto.make_env()","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:37:48.413633Z","iopub.execute_input":"2021-11-05T11:37:48.414325Z","iopub.status.idle":"2021-11-05T11:37:48.450566Z","shell.execute_reply.started":"2021-11-05T11:37:48.414270Z","shell.execute_reply":"2021-11-05T11:37:48.449632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Pandas Profiling","metadata":{}},{"cell_type":"code","source":"from pandas_profiling import ProfileReport","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:37:48.452011Z","iopub.execute_input":"2021-11-05T11:37:48.452947Z","iopub.status.idle":"2021-11-05T11:37:49.462540Z","shell.execute_reply.started":"2021-11-05T11:37:48.452899Z","shell.execute_reply":"2021-11-05T11:37:49.461409Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report = ProfileReport(df2)\nreport","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:37:49.464432Z","iopub.execute_input":"2021-11-05T11:37:49.464806Z","iopub.status.idle":"2021-11-05T11:38:20.383907Z","shell.execute_reply.started":"2021-11-05T11:37:49.464759Z","shell.execute_reply":"2021-11-05T11:38:20.381205Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"report = ProfileReport(df_asset)\nreport","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:38:20.385370Z","iopub.execute_input":"2021-11-05T11:38:20.385832Z","iopub.status.idle":"2021-11-05T11:38:25.398203Z","shell.execute_reply.started":"2021-11-05T11:38:20.385785Z","shell.execute_reply":"2021-11-05T11:38:25.396555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # Feature Selection","metadata":{}},{"cell_type":"code","source":"X = df.drop('Target',axis = 1)\nY = df['Target']\nX_train,X_test,Y_train,Y_test = train_test_split(X,Y,test_size = 0.3,random_state=44)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:38:25.400170Z","iopub.execute_input":"2021-11-05T11:38:25.400587Z","iopub.status.idle":"2021-11-05T11:38:32.333800Z","shell.execute_reply.started":"2021-11-05T11:38:25.400538Z","shell.execute_reply":"2021-11-05T11:38:32.332707Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> # LGBMRegression prediction","metadata":{}},{"cell_type":"code","source":"import lightgbm as ltb\nmodel = ltb.LGBMRegressor(n_estimators=10,learning_rate=0.09)\nmodel.fit(X_train, Y_train)\nprint(); print(model) \nexpected_y = Y_test\npredicted_y = model.predict(X_test)","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:38:32.335931Z","iopub.execute_input":"2021-11-05T11:38:32.336295Z","iopub.status.idle":"2021-11-05T11:38:54.804067Z","shell.execute_reply.started":"2021-11-05T11:38:32.336223Z","shell.execute_reply":"2021-11-05T11:38:54.803065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(metrics.r2_score(expected_y, predicted_y))","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:38:54.805655Z","iopub.execute_input":"2021-11-05T11:38:54.806013Z","iopub.status.idle":"2021-11-05T11:38:54.919261Z","shell.execute_reply.started":"2021-11-05T11:38:54.805964Z","shell.execute_reply":"2021-11-05T11:38:54.918192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred1 = model.predict(df2)\npred1","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:38:54.920694Z","iopub.execute_input":"2021-11-05T11:38:54.921313Z","iopub.status.idle":"2021-11-05T11:38:54.934084Z","shell.execute_reply.started":"2021-11-05T11:38:54.921253Z","shell.execute_reply":"2021-11-05T11:38:54.932769Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"iter_test = env.iter_test() # an iterator which loops over the test set and sample submission\nfor (df2, sample_prediction_df) in iter_test:\n model = ltb.LGBMRegressor(n_estimators=10,learning_rate=0.09)\n model.fit(X_train, Y_train)\n df2.drop(['row_id'],axis=1,inplace = True)\n pred2 = model.predict(df2)\n sample_prediction_df['Target'] = pred2 # make your predictions here\n env.predict(sample_prediction_df) # register your predictions","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:38:54.935762Z","iopub.execute_input":"2021-11-05T11:38:54.936993Z","iopub.status.idle":"2021-11-05T11:40:05.685299Z","shell.execute_reply.started":"2021-11-05T11:38:54.936941Z","shell.execute_reply":"2021-11-05T11:40:05.684198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_prediction_df","metadata":{"execution":{"iopub.status.busy":"2021-11-05T11:40:05.687303Z","iopub.execute_input":"2021-11-05T11:40:05.687694Z","iopub.status.idle":"2021-11-05T11:40:05.702090Z","shell.execute_reply.started":"2021-11-05T11:40:05.687633Z","shell.execute_reply":"2021-11-05T11:40:05.700680Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}