{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#                                                 **FINAL MODEL+EDA+OBSERVATIONS+ENSEMBLE**","metadata":{}},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy import stats\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nimport warnings\nwarnings.filterwarnings(\"ignore\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:19.499604Z","iopub.execute_input":"2021-09-01T16:36:19.501402Z","iopub.status.idle":"2021-09-01T16:36:20.713337Z","shell.execute_reply.started":"2021-09-01T16:36:19.501221Z","shell.execute_reply":"2021-09-01T16:36:20.712034Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_columns=100\nsmall=0.00001 #to prevent division by zero","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:20.715300Z","iopub.execute_input":"2021-09-01T16:36:20.715752Z","iopub.status.idle":"2021-09-01T16:36:20.721944Z","shell.execute_reply.started":"2021-09-01T16:36:20.715692Z","shell.execute_reply":"2021-09-01T16:36:20.720635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df=pd.read_csv(\"../input/beyond-analysis/train.csv\")\ntest_df=pd.read_csv(\"../input/beyond-analysis/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:20.724568Z","iopub.execute_input":"2021-09-01T16:36:20.725356Z","iopub.status.idle":"2021-09-01T16:36:27.900150Z","shell.execute_reply.started":"2021-09-01T16:36:20.725277Z","shell.execute_reply":"2021-09-01T16:36:27.898937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Lets check shape of data**","metadata":{}},{"cell_type":"code","source":"print(\"shape of train:\",train_df.shape)\nprint(\"shape of test:\",test_df.shape)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:27.902210Z","iopub.execute_input":"2021-09-01T16:36:27.902673Z","iopub.status.idle":"2021-09-01T16:36:27.910435Z","shell.execute_reply.started":"2021-09-01T16:36:27.902639Z","shell.execute_reply":"2021-09-01T16:36:27.909206Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Check NULL values in our data**","metadata":{}},{"cell_type":"code","source":"# check if there are null values or not\ntrain_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:27.912031Z","iopub.execute_input":"2021-09-01T16:36:27.912797Z","iopub.status.idle":"2021-09-01T16:36:28.229466Z","shell.execute_reply.started":"2021-09-01T16:36:27.912746Z","shell.execute_reply":"2021-09-01T16:36:28.227957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Check NULL values in our data**","metadata":{}},{"cell_type":"code","source":"# check if there are null values or not\ntest_df.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:28.231833Z","iopub.execute_input":"2021-09-01T16:36:28.232373Z","iopub.status.idle":"2021-09-01T16:36:28.459814Z","shell.execute_reply.started":"2021-09-01T16:36:28.232323Z","shell.execute_reply":"2021-09-01T16:36:28.458589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Check data types of values values in our data**","metadata":{}},{"cell_type":"code","source":"train_df.info()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:28.461674Z","iopub.execute_input":"2021-09-01T16:36:28.462445Z","iopub.status.idle":"2021-09-01T16:36:28.819624Z","shell.execute_reply.started":"2021-09-01T16:36:28.462391Z","shell.execute_reply":"2021-09-01T16:36:28.818296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Lets gain information about the columns to be predicted**","metadata":{}},{"cell_type":"code","source":"print(\"the maximum value in the column Y1 is:\",train_df['Y1'].max())\nprint(\"the minimum value in the column Y1 is:\",train_df['Y1'].min())\nprint(\"the mean value in the column Y1 is:\",train_df['Y1'].mean())\nprint(\"the standard deviation value in the column Y1 is:\",train_df['Y1'].std())\nprint('-'*100)\nprint(\"the maximum value in the column Y1 is:\",train_df['Y2'].max())\nprint(\"the minimum value in the column Y1 is:\",train_df['Y2'].min())\nprint(\"the mean value in the column Y1 is:\",train_df['Y2'].mean())\nprint(\"the standard deviation value in the column Y1 is:\",train_df['Y2'].std())","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:28.823929Z","iopub.execute_input":"2021-09-01T16:36:28.824761Z","iopub.status.idle":"2021-09-01T16:36:28.864408Z","shell.execute_reply.started":"2021-09-01T16:36:28.824707Z","shell.execute_reply":"2021-09-01T16:36:28.863123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data=train_df.groupby(['UNIQUE_IDENTIFIER'])['Y1','Y2'].mean()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:28.867169Z","iopub.execute_input":"2021-09-01T16:36:28.867884Z","iopub.status.idle":"2021-09-01T16:36:28.960108Z","shell.execute_reply.started":"2021-09-01T16:36:28.867830Z","shell.execute_reply":"2021-09-01T16:36:28.958881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('-'*100)\nprint(\"the maximum value in the column Y1 is:\",data['Y1'].max())\nprint(\"the minimum value in the column Y1 is:\",data['Y1'].min())\nprint(\"the mean value in the column Y1 is:\",data['Y1'].mean())\nprint(\"the standard deviation value in the column Y1 is:\",data['Y1'].std())\nprint('-'*100)\nprint(\"the maximum value in the column Y1 is:\",data['Y2'].max())\nprint(\"the minimum value in the column Y1 is:\",data['Y2'].min())\nprint(\"the mean value in the column Y1 is:\",data['Y2'].mean())\nprint(\"the standard deviation value in the column Y1 is:\",data['Y2'].std())\nprint('-'*100)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:28.961851Z","iopub.execute_input":"2021-09-01T16:36:28.962403Z","iopub.status.idle":"2021-09-01T16:36:28.982014Z","shell.execute_reply.started":"2021-09-01T16:36:28.962355Z","shell.execute_reply":"2021-09-01T16:36:28.980390Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **OBSERVATIONS**\n* there is a huge standard deviation in our Y2 column so with reference to the evaluation metric of the competition it is more important to reduce the RMSE of Y2 metric than Y1 metric\n* so we recieved a significant improvement in the variance of both columns if we group the data on the basis of unique identifier.\n* this will lead to loss of information therefore the main crux we have to do is how can we retain a lot of information for my model to perform better we can use min, max, mean, std values, appart from that we can use a lot amount of feature engineering which is done below","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**OBSERVATIONS** \n* **Y1 and Y2 are labels to be predicted** \n* **This is a r problem**\n* **there are no missing values in test and train set**","metadata":{}},{"cell_type":"code","source":"#display(train_df.target.describe())\nf, ax = plt.subplots(nrows=2, ncols=3, figsize=(18, 4))\nsns.distplot(train_df.Y1, ax=ax[0,0])\nsns.boxplot(train_df.Y1, ax=ax[0,1])\nstats.probplot(train_df['Y1'], plot=ax[0,2])\n\nsns.distplot(train_df.Y2, ax=ax[1,0])\nsns.boxplot(train_df.Y2, ax=ax[1,1])\nstats.probplot(train_df['Y2'], plot=ax[1,2])\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:28.984912Z","iopub.execute_input":"2021-09-01T16:36:28.985688Z","iopub.status.idle":"2021-09-01T16:36:48.061657Z","shell.execute_reply.started":"2021-09-01T16:36:28.985636Z","shell.execute_reply":"2021-09-01T16:36:48.060456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **OBSERVATIONS:**\n* There is positively skewed skewness in the metrices to be predicted,the reason is for a particular id there are many sparse values so this also indicates that we should try grouping our data to remove the skewness.\n* for metrix Y2 prob plot showed logistic distribution between observed and theoritical values\n* There are huge  outliers in our data (for ex there is a value above 800 that should be removed).","metadata":{}},{"cell_type":"code","source":"features = [feature for feature in train_df.columns if feature not in ['UNIQUE_IDENTIFIER', 'Y1','Y2','CATEGORY_1','CATEGORY_2']]\n\nfig = plt.figure(figsize=(12, 12), facecolor='#f6f6f6')\ngs = fig.add_gridspec(5, 4)\ngs.update(wspace=0.1, hspace=0.4)\n\nbackground_color = \"#f6f6f6\"\n\nrun_no = 0\nfor row in range(0, 5):\n    for col in range(0, 4):\n        locals()[\"ax\"+str(run_no)] = fig.add_subplot(gs[row, col])\n        locals()[\"ax\"+str(run_no)].set_facecolor(background_color)\n        locals()[\"ax\"+str(run_no)].tick_params(axis='y', left=False)\n        locals()[\"ax\"+str(run_no)].get_yaxis().set_visible(False)\n        for s in [\"top\",\"right\",\"left\"]:\n            locals()[\"ax\"+str(run_no)].spines[s].set_visible(False)\n        run_no += 1\n\nrun_no = 0\nfor feature in features:\n        sns.kdeplot(train_df[feature] ,ax=locals()[\"ax\"+str(run_no)], color='#ffd514', shade=True, linewidth=1.5, alpha=0.9, zorder=3, legend=False)\n        locals()[\"ax\"+str(run_no)].grid(which='major', axis='x', zorder=0, color='gray', linestyle=':', dashes=(1,5))\n        locals()[\"ax\"+str(run_no)].set_xlabel(feature)\n        run_no += 1\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:36:48.063358Z","iopub.execute_input":"2021-09-01T16:36:48.064022Z","iopub.status.idle":"2021-09-01T16:38:30.736290Z","shell.execute_reply.started":"2021-09-01T16:36:48.063972Z","shell.execute_reply":"2021-09-01T16:38:30.734966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **OBSERVATIONS:**\n* all the continuous features are also left skewed they reason is obvioiusly because of huge amount of 0 values present in the data.\n* we can try to remove this skewness by converting the scaling the data with the help of quantile transformer.(in processing section).\n* also since it is required that we have to predict one value for each unique identifier we can also group the data which significantly reduces the 0 values.","metadata":{}},{"cell_type":"code","source":"f, ax = plt.subplots(nrows=1, ncols=1, figsize=(20, 20))\nax.set_title(\"Correlation Matrix\", fontsize=16)\nsns.heatmap(train_df[train_df.columns[train_df.columns != 'UNIQUE_IDENTIFIER']].corr(), vmin=-1, vmax=1, cmap='coolwarm', annot=True)\n\nfor tick in ax.xaxis.get_major_ticks():\n    tick.label.set_fontsize(14) \n    tick.label.set_rotation(90) \nfor tick in ax.yaxis.get_major_ticks():\n    tick.label.set_fontsize(14)\n    tick.label.set_rotation(0) \nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:30.737820Z","iopub.execute_input":"2021-09-01T16:38:30.738272Z","iopub.status.idle":"2021-09-01T16:38:35.986715Z","shell.execute_reply.started":"2021-09-01T16:38:30.738227Z","shell.execute_reply":"2021-09-01T16:38:35.985559Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CORRELATION OBSERVATIONS**\n* there is sinificant correlation between Y2 and sequence size and status check.\n* There seems to be huge correlation between **ENTRY ,REVENUE AND WINNINGS_1**\n* Y2 seems to be fairly correlated with **SEQUENCE_NO,STATUS_CHECK** \n* there seems to be fair correlation between variouse columns hence combined polynomial featues can be build \n","metadata":{}},{"cell_type":"markdown","source":"# **Categorical Features** ","metadata":{}},{"cell_type":"code","source":"print(\"unique values in category 1 are:(train set) \",train_df['CATEGORY_1'].unique())\nprint(\"unique values in category 1 are(test set): \",test_df['CATEGORY_1'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:35.988368Z","iopub.execute_input":"2021-09-01T16:38:35.988991Z","iopub.status.idle":"2021-09-01T16:38:36.175753Z","shell.execute_reply.started":"2021-09-01T16:38:35.988948Z","shell.execute_reply":"2021-09-01T16:38:36.174457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"unique values in category 2 are:(train set) \",train_df['CATEGORY_2'].unique())\nprint(\"unique values in category 2 are(test set): \",test_df['CATEGORY_2'].unique())","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:36.177448Z","iopub.execute_input":"2021-09-01T16:38:36.178181Z","iopub.status.idle":"2021-09-01T16:38:36.354914Z","shell.execute_reply.started":"2021-09-01T16:38:36.178128Z","shell.execute_reply":"2021-09-01T16:38:36.353469Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **OBSERVATIONS**\n* Not all the values in train and test data are same.\n\n* We can perform simple label encoding on the categorical features.","metadata":{}},{"cell_type":"markdown","source":"# **FEATURE ENGINEERING/EXTRACTION**","metadata":{}},{"cell_type":"code","source":"# corr_featu=[\"DEPOSIT\", \"ENTRY\", \"REVENUE\", \"WINNINGS_1\"]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:36.357252Z","iopub.execute_input":"2021-09-01T16:38:36.357763Z","iopub.status.idle":"2021-09-01T16:38:36.364093Z","shell.execute_reply.started":"2021-09-01T16:38:36.357706Z","shell.execute_reply":"2021-09-01T16:38:36.362648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#function to find the difference in the series\ndef differ(Series):\n    return Series.diff()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:36.366355Z","iopub.execute_input":"2021-09-01T16:38:36.366831Z","iopub.status.idle":"2021-09-01T16:38:36.378276Z","shell.execute_reply.started":"2021-09-01T16:38:36.366784Z","shell.execute_reply":"2021-09-01T16:38:36.376666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Description of columns**\n* pct is simply the difference between deposit and entry(important).\n* fract is the ratio fun contest wins and the fun contests participated.\n* fract1 is the ratio  contest wins and the fun contests participated(very important).\n* fract2 is the ratio  contest wins and the fun contests participated.\n* paid is difference of revenue and discount\n* fract4 is difference of  entry and winnings1(total chips wins)","metadata":{}},{"cell_type":"code","source":"train_df[train_df['DISCOUNT']<=train_df['ENTRY']]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:36.380642Z","iopub.execute_input":"2021-09-01T16:38:36.381145Z","iopub.status.idle":"2021-09-01T16:38:36.573613Z","shell.execute_reply.started":"2021-09-01T16:38:36.381097Z","shell.execute_reply":"2021-09-01T16:38:36.572067Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# difference in the cards to be purchased an actual cards purchased\ntrain_df['pct']=(train_df['DEPOSIT']-train_df['ENTRY'])\ntest_df['pct']=(test_df['DEPOSIT']-test_df['ENTRY']) \n#\ntrain_df['fract']=train_df['PRACTICE_WINNINGS_NUMBER']/(train_df['PRACTICE_ENTRY_NUMBER']+0.00001)\ntest_df['fract']=test_df['PRACTICE_WINNINGS_NUMBER']/(test_df['PRACTICE_ENTRY_NUMBER']+0.00001)\n# ENTRY_NUMBER - Contests participated\n# WINNINGS_NUMBER - Contests won\ntrain_df['fract1']=train_df['WINNINGS_NUMBER']/(train_df['ENTRY_NUMBER']+0.00001)\ntest_df['fract1']=test_df['WINNINGS_NUMBER']/(test_df['ENTRY_NUMBER']+0.00001)\n\ntrain_df['fract2']=train_df['PRACTICE_WINNINGS']/(train_df['PRACTICE_ENTRY']+0.00001)\ntest_df['fract2']=test_df['PRACTICE_WINNINGS']/(test_df['PRACTICE_ENTRY']+0.00001)\n\ntrain_df['fract3']=(train_df['WITHDRAW']-train_df['DEPOSIT'])/(train_df['WITHDRAW']+train_df['DEPOSIT']+0.00001)\ntest_df['fract3']=(test_df['WITHDRAW']-test_df['DEPOSIT'])/(test_df['WITHDRAW']+test_df['DEPOSIT']+0.000001)\n\ntrain_df['paid']=train_df['REVENUE']-train_df['DISCOUNT']\ntest_df['paid']=test_df['REVENUE']-test_df['DISCOUNT']\n\ntrain_df['fract4']=train_df['ENTRY']-train_df['WINNINGS_1']\ntest_df['fract4']=test_df['ENTRY']-test_df['WINNINGS_1']\n\ntrain_df['fract5']=train_df['REVENUE']/(train_df['ENTRY']+0.01)\ntest_df['fract5']=test_df['REVENUE']/(test_df['ENTRY']+0.01)\n\ntrain_df['fract6']=train_df['DISCOUNT']/(train_df['ENTRY']+0.01)\ntest_df['fract6']=test_df['DISCOUNT']/(test_df['ENTRY']+0.01)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:36.576095Z","iopub.execute_input":"2021-09-01T16:38:36.576595Z","iopub.status.idle":"2021-09-01T16:38:36.730345Z","shell.execute_reply.started":"2021-09-01T16:38:36.576549Z","shell.execute_reply":"2021-09-01T16:38:36.729188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [feature for feature in train_df.columns if feature not in ['UNIQUE_IDENTIFIER','SEQUENCE_NO', 'Y1','Y2','CATEGORY_1','CATEGORY_2']]\nfor i in features:\n    train_df[i+'_diff']=differ(train_df[i])\n    test_df[i+'_diff']=differ(test_df[i])\n\nfor i in features:\n    train_df[i+'_diff'][train_df['SEQUENCE_NO']==1]=0.00\n    test_df[i+'_diff'][test_df['SEQUENCE_NO']==1]=0.00","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:36.732227Z","iopub.execute_input":"2021-09-01T16:38:36.732695Z","iopub.status.idle":"2021-09-01T16:38:37.434841Z","shell.execute_reply.started":"2021-09-01T16:38:36.732621Z","shell.execute_reply":"2021-09-01T16:38:37.433679Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[train_df['SEQUENCE_NO']==1]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:37.437212Z","iopub.execute_input":"2021-09-01T16:38:37.437883Z","iopub.status.idle":"2021-09-01T16:38:38.063912Z","shell.execute_reply.started":"2021-09-01T16:38:37.437834Z","shell.execute_reply":"2021-09-01T16:38:38.062523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **PREPROCESSING UNGROUPED DATA**","metadata":{}},{"cell_type":"code","source":"import string\ncat_columns=['CATEGORY_1', 'CATEGORY_2']\nfrom sklearn.preprocessing import LabelEncoder\nlb=LabelEncoder()\nlb.fit(list(string.ascii_uppercase))\nfor x in cat_columns:\n    train_df[x]=lb.transform(train_df[x])\n    test_df[x]=lb.transform(test_df[x])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:38.066216Z","iopub.execute_input":"2021-09-01T16:38:38.066703Z","iopub.status.idle":"2021-09-01T16:38:39.482348Z","shell.execute_reply.started":"2021-09-01T16:38:38.066643Z","shell.execute_reply":"2021-09-01T16:38:39.480953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **GROUPING**\n* Grouping the data on the basis of unique identifer, this is surely going to reduce the 0 values in the data and possibly introduce less skewed variables it helped me in reducing standard deviation.","metadata":{}},{"cell_type":"code","source":"def percentile(n):\n    def percentile_(x):\n        return np.percentile(x, n)\n    percentile_.__name__ = 'percentile_%s' % n\n    return percentile_\ni=[np.mean,np.max,np.min,np.sum,np.std,np.median]\n# i=[np.mean,np.max,np.min,np.sum,np.std,np.median]\nfe_dict = {\n        'SEQUENCE_NO':[np.size],\n        'STATUS_CHECK':[np.mean],\n        'CATEGORY_1':[np.mean],\n        'CATEGORY_2':[np.mean],\n        'ACTIVE_YN':[np.mean,np.sum],\n        'ENTRY':i,\n        'REVENUE':i,\n        'WINNINGS_1':i,\n        'WINNINGS_2':i,\n        'DISCOUNT':i,\n        'DEPOSIT':i,\n        'DEPOSIT_NUMBER':i,\n        'DEPOSIT_2':i,\n        'WITHDRAW':i,\n        'WITHDRAW_NUMBER':i,\n        'DEPOSIT_TRAILS':i,\n        'ENTRY_NUMBER':i,\n        'WINNINGS_NUMBER':i,\n        'PRACTICE_ENTRY':i,\n        'PRACTICE_WINNINGS':i,\n        'PRACTICE_ENTRY_NUMBER':i,\n        'PRACTICE_WINNINGS_NUMBER':i,\n        'ACTIVE_YN_diff':[np.mean,np.sum],\n        'ENTRY_diff':i,\n        'REVENUE_diff':i,\n        'WINNINGS_1_diff':i,\n         'WINNINGS_2_diff':i,\n        'DISCOUNT_diff':i,\n        'DEPOSIT_diff':i,\n        'DEPOSIT_NUMBER_diff':i,\n        'DEPOSIT_2_diff':i,\n        'WITHDRAW_diff':i,\n        'WITHDRAW_NUMBER_diff':i,\n        'DEPOSIT_TRAILS_diff':i,\n        'ENTRY_NUMBER_diff':i,\n        'WINNINGS_NUMBER_diff':i,\n        'PRACTICE_ENTRY_diff':i,\n        'PRACTICE_WINNINGS_diff':i,\n        'PRACTICE_ENTRY_NUMBER_diff':i,\n        'PRACTICE_WINNINGS_NUMBER_diff':i,\n        'pct':i,\n        'fract':i,\n        'fract1':i,\n        'fract2':i,\n        'fract3':i,\n        'fract4':i,\n        'fract5':i,\n        'fract6':i,\n        'paid':i,\n        'Y1':[np.mean],\n        'Y2':[np.mean],\n         }\ntrain=train_df.groupby(['UNIQUE_IDENTIFIER']).agg(fe_dict).reset_index()\ntrain.columns = ['_'.join(col) for col in train.columns]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:39.490158Z","iopub.execute_input":"2021-09-01T16:38:39.490660Z","iopub.status.idle":"2021-09-01T16:38:44.547395Z","shell.execute_reply.started":"2021-09-01T16:38:39.490612Z","shell.execute_reply":"2021-09-01T16:38:44.546225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f, ax = plt.subplots(nrows=1, ncols=1, figsize=(6,6))\n#f.suptitle('Distribution of Features', fontsize=16)\nsns.distplot(train['DEPOSIT_sum'])\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:44.550284Z","iopub.execute_input":"2021-09-01T16:38:44.550617Z","iopub.status.idle":"2021-09-01T16:38:45.497889Z","shell.execute_reply.started":"2021-09-01T16:38:44.550584Z","shell.execute_reply":"2021-09-01T16:38:45.491900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fe_dict = {\n        'SEQUENCE_NO':[np.size],\n        'STATUS_CHECK':[np.mean],\n        'CATEGORY_1':[np.mean],\n        'CATEGORY_2':[np.mean],\n        'ACTIVE_YN':[np.mean,np.sum],\n        'ENTRY':i,\n        'REVENUE':i,\n        'WINNINGS_1':i,\n        'WINNINGS_2':i,\n        'DISCOUNT':i,\n        'DEPOSIT':i,\n        'DEPOSIT_NUMBER':i,\n        'DEPOSIT_2':i,\n        'WITHDRAW':i,\n        'WITHDRAW_NUMBER':i,\n        'DEPOSIT_TRAILS':i,\n        'ENTRY_NUMBER':i,\n        'WINNINGS_NUMBER':i,\n        'PRACTICE_ENTRY':i,\n        'PRACTICE_WINNINGS':i,\n        'PRACTICE_ENTRY_NUMBER':i,\n        'PRACTICE_WINNINGS_NUMBER':i,\n        'ACTIVE_YN_diff':[np.mean,np.sum],\n        'ENTRY_diff':i,\n        'REVENUE_diff':i,\n        'WINNINGS_1_diff':i,\n         'WINNINGS_2_diff':i,\n        'DISCOUNT_diff':i,\n        'DEPOSIT_diff':i,\n        'DEPOSIT_NUMBER_diff':i,\n        'DEPOSIT_2_diff':i,\n        'WITHDRAW_diff':i,\n        'WITHDRAW_NUMBER_diff':i,\n        'DEPOSIT_TRAILS_diff':i,\n        'ENTRY_NUMBER_diff':i,\n        'WINNINGS_NUMBER_diff':i,\n        'PRACTICE_ENTRY_diff':i,\n        'PRACTICE_WINNINGS_diff':i,\n        'PRACTICE_ENTRY_NUMBER_diff':i,\n        'PRACTICE_WINNINGS_NUMBER_diff':i,\n        'pct':i,\n        'fract':i,\n        'fract1':i,\n        'fract2':i,\n         'fract3':i,\n        'fract4':i,\n        'fract5':i,\n        'fract6':i,\n        'paid':i,\n         }\ntest=test_df.groupby(['UNIQUE_IDENTIFIER']).agg(fe_dict).reset_index()\ntest.columns = ['_'.join(col) for col in test.columns]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:45.500439Z","iopub.execute_input":"2021-09-01T16:38:45.500947Z","iopub.status.idle":"2021-09-01T16:38:49.185209Z","shell.execute_reply.started":"2021-09-01T16:38:45.500898Z","shell.execute_reply":"2021-09-01T16:38:49.183998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:49.187449Z","iopub.execute_input":"2021-09-01T16:38:49.188042Z","iopub.status.idle":"2021-09-01T16:38:49.502405Z","shell.execute_reply.started":"2021-09-01T16:38:49.187960Z","shell.execute_reply":"2021-09-01T16:38:49.500896Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# STATUS_CHECK_mean SEQUENCE_NO_size\ntrain['fract1_']=train['WINNINGS_NUMBER_sum']/(train['ENTRY_NUMBER_sum']+0.00001)\ntest['fract1_']=test['WINNINGS_NUMBER_sum']/(test['ENTRY_NUMBER_sum']+0.00001)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:49.504455Z","iopub.execute_input":"2021-09-01T16:38:49.505210Z","iopub.status.idle":"2021-09-01T16:38:49.518766Z","shell.execute_reply.started":"2021-09-01T16:38:49.505161Z","shell.execute_reply":"2021-09-01T16:38:49.517054Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train=train.fillna(0.0)\ntest=test.fillna(0.0)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:49.520936Z","iopub.execute_input":"2021-09-01T16:38:49.521821Z","iopub.status.idle":"2021-09-01T16:38:50.026472Z","shell.execute_reply.started":"2021-09-01T16:38:49.521772Z","shell.execute_reply":"2021-09-01T16:38:50.025127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features=['Y1_mean','Y2_mean']","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:50.028335Z","iopub.execute_input":"2021-09-01T16:38:50.028863Z","iopub.status.idle":"2021-09-01T16:38:50.035163Z","shell.execute_reply.started":"2021-09-01T16:38:50.028810Z","shell.execute_reply":"2021-09-01T16:38:50.033523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nkmeans = KMeans(n_clusters=10, random_state=2021)\nkmeans.fit(train[features])\n\npca = PCA(n_components=2)\nstocks_2d = pca.fit_transform(train[features])\n\nfig, ax = plt.subplots(figsize=(32, 10))\nax.scatter(stocks_2d[:, 0], stocks_2d[:, 1], s=200, c=kmeans.labels_, cmap='RdBu')\n# for idx, stock_id in enumerate(train['UNIQUE_IDENTIFIER_'].values):\n#     ax.annotate(stock_id, (stocks_2d[idx, 0], stocks_2d[idx, 1]), fontsize=20)\n    \nax.tick_params(axis='x', labelsize=20, pad=10)\nax.tick_params(axis='y', labelsize=20, pad=10)\nax.set_title('ID Clusters', size=25, pad=20)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:50.037466Z","iopub.execute_input":"2021-09-01T16:38:50.038563Z","iopub.status.idle":"2021-09-01T16:38:56.981510Z","shell.execute_reply.started":"2021-09-01T16:38:50.038514Z","shell.execute_reply":"2021-09-01T16:38:56.980347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# OBSERVATIONS:\n*  based on y1 and y2 there are outliers in our data.\n*  I had an idea to produce startified kfold based on clusters i performed it but it didn't worked well so i didn't proceed with it","metadata":{}},{"cell_type":"code","source":"print(\"the maximum value in the column Y1 is:\",train['Y1_mean'].max())\nprint(\"the minimum value in the column Y1 is:\",train['Y1_mean'].min())\nprint(\"the mean value in the column Y1 is:\",train['Y1_mean'].mean())\nprint(\"the standard deviation value in the column Y1 is:\",train['Y1_mean'].std())\nprint('-'*100)\nprint(\"the maximum value in the column Y1 is:\",train['Y2_mean'].max())\nprint(\"the minimum value in the column Y1 is:\",train['Y2_mean'].min())\nprint(\"the mean value in the column Y1 is:\",train['Y2_mean'].mean())\nprint(\"the standard deviation value in the column Y1 is:\",train['Y2_mean'].std())","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:56.983222Z","iopub.execute_input":"2021-09-01T16:38:56.983896Z","iopub.status.idle":"2021-09-01T16:38:57.005135Z","shell.execute_reply.started":"2021-09-01T16:38:56.983850Z","shell.execute_reply":"2021-09-01T16:38:57.003884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n# **Scaling Data**\n\n* **QuantileTransformer**\n\nTransform features using quantiles information.\n\nThis method transforms the features to follow a uniform or a normal distribution. Therefore, for a given feature, this transformation tends to spread out the most frequent values. It also reduces the impact of (marginal) outliers: this is therefore a robust preprocessing scheme.\n\nThe transformation is applied on each feature independently. First an estimate of the cumulative distribution function of a feature is used to map the original values to a uniform distribution. The obtained values are then mapped to the desired output distribution using the associated quantile function. Features values of new/unseen data that fall below or above the fitted range will be mapped to the bounds of the output distribution. Note that this transform is non-linear. It may distort linear correlations between variables measured at the same scale but renders variables measured at different scales more directly comparable.\n","metadata":{}},{"cell_type":"code","source":"features = [col for col in train.columns if col not in {\"Y1_mean\",\"Y2_mean\",\"UNIQUE_IDENTIFIER_\"}]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:57.006696Z","iopub.execute_input":"2021-09-01T16:38:57.007372Z","iopub.status.idle":"2021-09-01T16:38:57.013819Z","shell.execute_reply.started":"2021-09-01T16:38:57.007307Z","shell.execute_reply":"2021-09-01T16:38:57.012406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Train\nfrom sklearn.preprocessing import QuantileTransformer\ntrain_model_sc = train.copy()\ntrain_label = train_model_sc[[\"Y1_mean\",\"Y2_mean\"]]\ntrain_model_sc = train_model_sc[features]\ncolumns = train_model_sc.columns\n# Test\ntest_model_sc = test.copy()\ntest_model_sc = test_model_sc[features]\n#Scaler\nqt = QuantileTransformer(n_quantiles=1000, random_state=0,output_distribution = \"normal\")\n# Scaling\ntrain[features] = qt.fit_transform(train_model_sc)\ntest[features]  = qt.transform(test_model_sc)\n\ntrain=train.fillna(0.0)\ntest=test.fillna(0.0)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:38:57.015723Z","iopub.execute_input":"2021-09-01T16:38:57.016655Z","iopub.status.idle":"2021-09-01T16:39:18.445819Z","shell.execute_reply.started":"2021-09-01T16:38:57.016597Z","shell.execute_reply":"2021-09-01T16:39:18.444426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **MODEL PREPARATION**","metadata":{}},{"cell_type":"code","source":"cont_features =[col for col in train.columns if col not in {\"Y1_mean\",\"Y2_mean\",\"UNIQUE_IDENTIFIER_\"}]\ncat_features = []\ntarget = train[\"Y1_mean\"]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:39:18.447547Z","iopub.execute_input":"2021-09-01T16:39:18.448055Z","iopub.status.idle":"2021-09-01T16:39:18.454112Z","shell.execute_reply.started":"2021-09-01T16:39:18.448010Z","shell.execute_reply":"2021-09-01T16:39:18.452613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Description\n* To deal with categorical features i made a pipeline containing leave one out encoder and label encoder however i didnt felt the need of using it since all the features i used were continuous except category1 and category2.I used label encoder before grouping the data on the categorical features.","metadata":{}},{"cell_type":"code","source":"# from category_encoders import LeaveOneOutEncoder\n# from sklearn.preprocessing import LabelEncoder\n\nxgb_cat_features = []\nlgb_cat_features = []\ncb_cat_features = []\nridge_cat_features = []\nsgd_cat_features = []\nhgbc_cat_features = []\n\n# loo_features = []\n# le_features = []\n# def loo_encode(train_df, test_df, column):\n#     loo = LeaveOneOutEncoder()\n#     new_feature = \"{}_loo\".format(column)\n#     loo.fit(train[column], target)\n#     train[column] = loo.transform(train[column])\n#     test[column] = loo.transform(test[column])\n#     return new_feature\n\n# for feature in cat_features:\n#     loo_features.append(loo_encode(train, test, feature))\n# #     le_features.append(label_encode(train, test, feature))\n    \n# # xgb_cat_features.extend(loo_features)\n# # lgb_cat_features.extend(le_features)\n# cb_cat_features.extend(cat_features)\n# # ridge_cat_features.extend(loo_features)\n# # sgd_cat_features.extend(loo_features)\n# # hgbc_cat_features.extend(loo_features)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:39:18.455825Z","iopub.execute_input":"2021-09-01T16:39:18.456647Z","iopub.status.idle":"2021-09-01T16:39:18.473308Z","shell.execute_reply.started":"2021-09-01T16:39:18.456498Z","shell.execute_reply":"2021-09-01T16:39:18.472069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELS DESCRIPTION\n* four models were used and there output was averaged\n>      1.   lightgbm regressor\n>      2.   catboost regressor\n>      3.   xgboost regressor\n>      4.   hyper gradient boosting regressor\n* one pipeline was for predicting Y1 and other for prediction Y2\n","metadata":{}},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport pandas\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDRegressor\n# from sklearn.calibration import CalibratedRegressorCV\n\nfrom sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingRegressor\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\n\ndef rmse(y_true, y_pred):\n    return np.sqrt(np.mean(np.square((y_true - y_pred))))\ndef feval_rmse(y_pred, lgb_train):\n    y_true = lgb_train.get_label()\n    return 'RMSE', rmse(y_true, y_pred), False\nrandom_state = 2021\nn_folds = 10\nk_fold = KFold(n_splits=n_folds, random_state=random_state, shuffle=True)\n\nxgb_train_preds = np.zeros(len(train.index), )\nxgb_test_preds = np.zeros(len(test.index), )\nxgb_features = xgb_cat_features + cont_features\n\nlgb_train_preds = np.zeros(len(train.index), )\nlgb_test_preds = np.zeros(len(test.index), )\nlgb_features = lgb_cat_features + cont_features\n\ncb_train_preds = np.zeros(len(train.index), )\ncb_test_preds = np.zeros(len(test.index), )\ncb_features = cb_cat_features + cont_features\n\n# ridge_train_preds = np.zeros(len(train.index), )\n# ridge_test_preds = np.zeros(len(test.index), )\n# ridge_features = ridge_cat_features + cont_features\n\n# sgd_train_preds = np.zeros(len(train.index), )\n# sgd_test_preds = np.zeros(len(test.index), )\n# sgd_features = sgd_cat_features + cont_features\n\nhgbc_train_preds = np.zeros(len(train.index), )\nhgbc_test_preds = np.zeros(len(test.index), )\nhgbc_features = hgbc_cat_features + cont_features\n\nfor fold, (train_index, test_index) in enumerate(k_fold.split(train)):\n    print(\"--> Fold {}\".format(fold + 1))\n    y_train = target.iloc[train_index]\n    y_valid = target.iloc[test_index]\n\n    xgb_x_train = pandas.DataFrame(train[xgb_features].iloc[train_index])\n    xgb_x_valid = pandas.DataFrame(train[xgb_features].iloc[test_index])\n\n    lgb_x_train = pandas.DataFrame(train[lgb_features].iloc[train_index])\n    lgb_x_valid = pandas.DataFrame(train[lgb_features].iloc[test_index])\n\n    cb_x_train = pandas.DataFrame(train[cb_features].iloc[train_index])\n    cb_x_valid = pandas.DataFrame(train[cb_features].iloc[test_index])\n\n#     ridge_x_train = pandas.DataFrame(train[ridge_features].iloc[train_index])\n#     ridge_x_valid = pandas.DataFrame(train[ridge_features].iloc[test_index])\n\n#     sgd_x_train = pandas.DataFrame(train[sgd_features].iloc[train_index])\n#     sgd_x_valid = pandas.DataFrame(train[sgd_features].iloc[test_index])\n\n    hgbc_x_train = pandas.DataFrame(train[hgbc_features].iloc[train_index])\n    hgbc_x_valid = pandas.DataFrame(train[hgbc_features].iloc[test_index])\n\n    xgb_model = XGBRegressor(\n        seed=2021,\n        n_estimators=5000,\n        verbosity=1,\n        eval_metric=\"rmse\",\n        tree_method=\"gpu_hist\",\n        gpu_id=0,\n        alpha=7.105038963844129,\n        colsample_bytree=0.25505629740052566,\n        gamma=0.4999381950212869,\n        reg_lambda=1.7256912198205319,\n        learning_rate=0.011823142071967673,\n        max_bin=338,\n        max_depth=8,\n        min_child_weight=2.286836198630466,\n        subsample=0.618417952155855,\n    )\n    xgb_model.fit(\n        xgb_x_train,\n        y_train,\n        eval_set=[(xgb_x_valid, y_valid)], \n        verbose=200,\n        early_stopping_rounds=500\n    )\n\n    train_oof_preds = xgb_model.predict(xgb_x_valid)\n    test_oof_preds = xgb_model.predict(test[xgb_features])\n    xgb_train_preds[test_index] = train_oof_preds\n    xgb_test_preds += test_oof_preds / n_folds\n    print(\": XGB - RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n    seed0=2021\n    params0 = {\n    'objective': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': -1,\n    'max_bin':100,\n    'min_data_in_leaf':500,\n    'learning_rate': 0.05,\n    'subsample': 0.72,\n    'subsample_freq': 4,\n    'feature_fraction': 0.5,\n    'lambda_l1': 0.5,\n    'lambda_l2': 1.0,\n    'categorical_column':[0],\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'n_jobs':-1,\n    'verbose': -1}\n    train_dataset = lgb.Dataset(lgb_x_train, y_train)\n    val_dataset = lgb.Dataset(lgb_x_valid, y_valid)\n    model = lgb.train(params = params0,\n                          num_boost_round=5000,\n                          train_set = train_dataset, \n                          valid_sets = [train_dataset, val_dataset], \n                          verbose_eval = 250,\n                          early_stopping_rounds=1000,\n                          feval = feval_rmse)\n    \n#     lgb.fit(\n#         lgb_x_train,\n#         y_train,\n#         eval_set=[(lgb_x_valid, y_valid)], \n#         verbose=200,\n#     )\n    train_oof_preds = model.predict(lgb_x_valid)\n    test_oof_preds = model.predict(test[lgb_features])\n    lgb_train_preds[test_index] = train_oof_preds\n    lgb_test_preds += test_oof_preds / n_folds\n    print(\": LGB- RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n\n    cb_model = CatBoostRegressor(\n        verbose=0,\n        eval_metric=\"RMSE\",\n        loss_function=\"RMSE\",\n        random_state=random_state,\n        num_boost_round=5000,\n        od_type=\"Iter\",\n        od_wait=1000,\n        task_type=\"GPU\",\n        devices=\"0\",\n        cat_features=[x for x in range(len(cb_cat_features))],\n        bagging_temperature=1.288692494969795,\n        grow_policy=\"Depthwise\",\n        l2_leaf_reg=9.847870133539244,\n        learning_rate=0.01877982653902465,\n        max_depth=8,\n        min_data_in_leaf=1,\n        penalties_coefficient=2.1176668909602734,\n    )\n    cb_model.fit(\n        cb_x_train,\n        y_train,\n        eval_set=[(cb_x_valid, y_valid)], \n        verbose=100,\n    )\n\n    train_oof_preds = cb_model.predict(cb_x_valid)\n    test_oof_preds = cb_model.predict(test[cb_features])\n    cb_train_preds[test_index] = train_oof_preds\n    cb_test_preds += test_oof_preds / n_folds\n    print(\": CATBOOST - RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n    \n#     ridge_model = CalibratedClassifierCV(\n#         RidgeClassifier(random_state=random_state),\n#         cv=3,\n#     )\n#     ridge_model.fit(\n#         ridge_x_train,\n#         y_train,\n#     )\n\n#     train_oof_preds = ridge_model.predict(ridge_x_valid)[:,-1]\n#     test_oof_preds = ridge_model.predict(test[ridge_features])\n#     ridge_train_preds[test_index] = train_oof_preds\n#     ridge_test_preds += test_oof_preds / n_folds\n#     print(\": Ridge - ROC AUC Score = {}\".format(roc_auc_score(y_valid, train_oof_preds, average=\"micro\")))\n    \n#     sgd_model = CalibratedClassifierCV(\n#         SGDClassifier(\n#             random_state=random_state,\n#             n_jobs=-1,\n#             loss=\"squared_hinge\",\n#         ),\n#         cv=3,\n#     )\n#     sgd_model.fit(\n#         sgd_x_train,\n#         y_train,\n#     )\n\n#     train_oof_preds = sgd_model.predict(sgd_x_valid)[:,-1]\n#     test_oof_preds = sgd_model.predict(test[sgd_features])[:,-1]\n#     sgd_train_preds[test_index] = train_oof_preds\n#     sgd_test_preds += test_oof_preds / n_folds\n#     print(\": SGD - ROC AUC Score = {}\".format(roc_auc_score(y_valid, train_oof_preds, average=\"micro\")))\n    \n    hgbc_model = HistGradientBoostingRegressor(\n        l2_regularization=1.766059063693552,\n        learning_rate=0.10675193678150449,\n        max_bins=128,\n        max_depth=31,\n        max_leaf_nodes=185,\n        random_state=2021\n    )\n    hgbc_model.fit(\n        hgbc_x_train,\n        y_train,\n    )\n\n    train_oof_preds = hgbc_model.predict(hgbc_x_valid)\n    test_oof_preds = hgbc_model.predict(test[hgbc_features])\n    hgbc_train_preds[test_index] = train_oof_preds\n    hgbc_test_preds += test_oof_preds / n_folds\n    print(\": HGBC- RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n    \n    \nprint(\"--> Overall metrics\")\nprint(\": XGB - rmse = {}\".format(rmse(target, xgb_train_preds)))\nprint(\": LGB - rmse = {}\".format(rmse(target, lgb_train_preds)))\nprint(\": CB - rmse = {}\".format(rmse(target, cb_train_preds)))\nprint(\": HGBC - rmse= {}\".format(rmse(target, hgbc_train_preds)))\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T16:39:18.475124Z","iopub.execute_input":"2021-09-01T16:39:18.475847Z","iopub.status.idle":"2021-09-01T17:08:16.561344Z","shell.execute_reply.started":"2021-09-01T16:39:18.475786Z","shell.execute_reply":"2021-09-01T17:08:16.559913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from scipy.special import expit\nfrom sklearn.calibration import CalibratedClassifierCV\n\nrandom_state = 2021\nn_folds = 10\n# k_fold =KFold(n_splits=n_folds, random_state=random_state, shuffle=True)\n\nl1_train = pandas.DataFrame(data={\n    \"xgb\": xgb_train_preds.tolist(),\n    \"lgb\": lgb_train_preds.tolist(),\n    \"cb\": cb_train_preds.tolist(),\n#     \"ridge\": ridge_train_preds.tolist(),\n#     \"sgd\": sgd_train_preds.tolist(),\n    \"hgbc\": hgbc_train_preds.tolist(),\n    \"target\": target.tolist()\n})\nl1_test = pandas.DataFrame(data={\n    \"xgb\": xgb_test_preds.tolist(),\n    \"lgb\": lgb_test_preds.tolist(),\n    \"cb\": cb_test_preds.tolist(),\n#     \"sgd\": sgd_test_preds.tolist(),\n#     \"ridge\": ridge_test_preds.tolist(),    \n    \"hgbc\": hgbc_test_preds.tolist(),\n})\n\ntrain_preds = np.zeros(len(l1_train.index), )\ntest_preds = np.zeros(len(l1_test.index), )\nfeatures = [\"xgb\", \"lgb\",\"cb\", \"hgbc\"]\nx_train = pandas.DataFrame(l1_test[features])\ntest_preds=(x_train[\"xgb\"]+x_train[\"lgb\"]+x_train[\"cb\"]+x_train[\"hgbc\"])/4","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:08:16.563562Z","iopub.execute_input":"2021-09-01T17:08:16.564342Z","iopub.status.idle":"2021-09-01T17:08:16.755872Z","shell.execute_reply.started":"2021-09-01T17:08:16.564287Z","shell.execute_reply":"2021-09-01T17:08:16.754685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pandas.read_csv(\"../input/beyond-analysis/sample_submission_random.csv\")\nsubmission[\"Y1\"] = test_preds.tolist()\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:08:16.757559Z","iopub.execute_input":"2021-09-01T17:08:16.758271Z","iopub.status.idle":"2021-09-01T17:08:16.935844Z","shell.execute_reply.started":"2021-09-01T17:08:16.758224Z","shell.execute_reply":"2021-09-01T17:08:16.934641Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntarget=train[\"Y2_mean\"]\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport pandas\nimport lightgbm as lgb\nfrom xgboost import XGBRegressor\nfrom catboost import CatBoostRegressor\nfrom sklearn.linear_model import Ridge\nfrom sklearn.linear_model import SGDRegressor\n# from sklearn.calibration import CalibratedRegressorCV\n\nfrom sklearn.experimental import enable_hist_gradient_boosting\nfrom sklearn.ensemble import HistGradientBoostingRegressor\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import roc_auc_score\n\n\ndef rmse(y_true, y_pred):\n    return np.sqrt(np.mean(np.square((y_true - y_pred))))\ndef feval_rmse(y_pred, lgb_train):\n    y_true = lgb_train.get_label()\n    return 'RMSE', rmse(y_true, y_pred), False\nrandom_state = 2021\nn_folds = 10\nk_fold = KFold(n_splits=n_folds, random_state=random_state, shuffle=True)\n\nxgb_train_preds = np.zeros(len(train.index), )\nxgb_test_preds = np.zeros(len(test.index), )\nxgb_features = xgb_cat_features + cont_features\n\nlgb_train_preds = np.zeros(len(train.index), )\nlgb_test_preds = np.zeros(len(test.index), )\nlgb_features = lgb_cat_features + cont_features\n\ncb_train_preds = np.zeros(len(train.index), )\ncb_test_preds = np.zeros(len(test.index), )\ncb_features = cb_cat_features + cont_features\n\nridge_train_preds = np.zeros(len(train.index), )\nridge_test_preds = np.zeros(len(test.index), )\nridge_features = ridge_cat_features + cont_features\n\nsgd_train_preds = np.zeros(len(train.index), )\nsgd_test_preds = np.zeros(len(test.index), )\nsgd_features = sgd_cat_features + cont_features\n\nhgbc_train_preds = np.zeros(len(train.index), )\nhgbc_test_preds = np.zeros(len(test.index), )\nhgbc_features = hgbc_cat_features + cont_features\n\nfor fold, (train_index, test_index) in enumerate(k_fold.split(train)):\n    print(\"--> Fold {}\".format(fold + 1))\n    y_train = target.iloc[train_index]\n    y_valid = target.iloc[test_index]\n\n    xgb_x_train = pandas.DataFrame(train[xgb_features].iloc[train_index])\n    xgb_x_valid = pandas.DataFrame(train[xgb_features].iloc[test_index])\n\n    lgb_x_train = pandas.DataFrame(train[lgb_features].iloc[train_index])\n    lgb_x_valid = pandas.DataFrame(train[lgb_features].iloc[test_index])\n\n    cb_x_train = pandas.DataFrame(train[cb_features].iloc[train_index])\n    cb_x_valid = pandas.DataFrame(train[cb_features].iloc[test_index])\n\n#     ridge_x_train = pandas.DataFrame(train[ridge_features].iloc[train_index])\n#     ridge_x_valid = pandas.DataFrame(train[ridge_features].iloc[test_index])\n\n#     sgd_x_train = pandas.DataFrame(train[sgd_features].iloc[train_index])\n#     sgd_x_valid = pandas.DataFrame(train[sgd_features].iloc[test_index])\n\n    hgbc_x_train = pandas.DataFrame(train[hgbc_features].iloc[train_index])\n    hgbc_x_valid = pandas.DataFrame(train[hgbc_features].iloc[test_index])\n\n    xgb_model = XGBRegressor(\n        seed=2021,\n        n_estimators=4000,\n        verbosity=1,\n        eval_metric=\"rmse\",\n        tree_method=\"gpu_hist\",\n        gpu_id=0,\n        alpha=7.105038963844129,\n        colsample_bytree=0.25505629740052566,\n        gamma=0.4999381950212869,\n        reg_lambda=1.7256912198205319,\n        learning_rate=0.011823142071967673,\n        max_bin=338,\n        max_depth=8,\n        min_child_weight=2.286836198630466,\n        subsample=0.618417952155855,\n    )\n    xgb_model.fit(\n        xgb_x_train,\n        y_train,\n        eval_set=[(xgb_x_valid, y_valid)], \n        verbose=200,\n        early_stopping_rounds=500\n    )\n\n    train_oof_preds = xgb_model.predict(xgb_x_valid)\n    test_oof_preds = xgb_model.predict(test[xgb_features])\n    xgb_train_preds[test_index] = train_oof_preds\n    xgb_test_preds += test_oof_preds / n_folds\n    print(\": XGB - RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n    seed0=2021\n    params0 = {\n    'objective': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': -1,\n    'max_bin':100,\n    'min_data_in_leaf':500,\n    'learning_rate': 0.05,\n    'subsample': 0.72,\n    'subsample_freq': 4,\n    'feature_fraction': 0.5,\n    'lambda_l1': 0.5,\n    'lambda_l2': 1.0,\n    'categorical_column':[0],\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'n_jobs':-1,\n    'verbose': -1}\n    train_dataset = lgb.Dataset(lgb_x_train, y_train)\n    val_dataset = lgb.Dataset(lgb_x_valid, y_valid)\n    model = lgb.train(params = params0,\n                          num_boost_round=5000,\n                          train_set = train_dataset, \n                          valid_sets = [train_dataset, val_dataset], \n                          verbose_eval = 250,\n                          early_stopping_rounds=1000,\n                          feval = feval_rmse)\n    \n#     lgb.fit(\n#         lgb_x_train,\n#         y_train,\n#         eval_set=[(lgb_x_valid, y_valid)], \n#         verbose=200,\n#     )\n    train_oof_preds = model.predict(lgb_x_valid)\n    test_oof_preds = model.predict(test[lgb_features])\n    lgb_train_preds[test_index] = train_oof_preds\n    lgb_test_preds += test_oof_preds / n_folds\n    print(\": lgb - RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n\n    cb_model = CatBoostRegressor(\n        verbose=0,\n        eval_metric=\"RMSE\",\n        loss_function=\"RMSE\",\n        random_state=random_state,\n        num_boost_round=5000,\n        od_type=\"Iter\",\n        od_wait=500,\n        task_type=\"GPU\",\n        devices=\"0\",\n        cat_features=[x for x in range(len(cb_cat_features))],\n        bagging_temperature=1.288692494969795,\n        grow_policy=\"Depthwise\",\n        l2_leaf_reg=9.847870133539244,\n        learning_rate=0.01877982653902465,\n        max_depth=8,\n        min_data_in_leaf=1,\n        penalties_coefficient=2.1176668909602734,\n    )\n    cb_model.fit(\n        cb_x_train,\n        y_train,\n        eval_set=[(cb_x_valid, y_valid)], \n        verbose=100,\n    )\n\n    train_oof_preds = cb_model.predict(cb_x_valid)\n    test_oof_preds = cb_model.predict(test[cb_features])\n    cb_train_preds[test_index] = train_oof_preds\n    cb_test_preds += test_oof_preds / n_folds\n    print(\": CATBOOST - RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n    \n#     ridge_model = CalibratedClassifierCV(\n#         RidgeClassifier(random_state=random_state),\n#         cv=3,\n#     )\n#     ridge_model.fit(\n#         ridge_x_train,\n#         y_train,\n#     )\n\n#     train_oof_preds = ridge_model.predict(ridge_x_valid)[:,-1]\n#     test_oof_preds = ridge_model.predict(test[ridge_features])\n#     ridge_train_preds[test_index] = train_oof_preds\n#     ridge_test_preds += test_oof_preds / n_folds\n#     print(\": Ridge - ROC AUC Score = {}\".format(roc_auc_score(y_valid, train_oof_preds, average=\"micro\")))\n    \n#     sgd_model = CalibratedClassifierCV(\n#         SGDClassifier(\n#             random_state=random_state,\n#             n_jobs=-1,\n#             loss=\"squared_hinge\",\n#         ),\n#         cv=3,\n#     )\n#     sgd_model.fit(\n#         sgd_x_train,\n#         y_train,\n#     )\n\n#     train_oof_preds = sgd_model.predict(sgd_x_valid)[:,-1]\n#     test_oof_preds = sgd_model.predict(test[sgd_features])[:,-1]\n#     sgd_train_preds[test_index] = train_oof_preds\n#     sgd_test_preds += test_oof_preds / n_folds\n#     print(\": SGD - ROC AUC Score = {}\".format(roc_auc_score(y_valid, train_oof_preds, average=\"micro\")))\n    \n    hgbc_model = HistGradientBoostingRegressor(\n        l2_regularization=1.766059063693552,\n        learning_rate=0.10675193678150449,\n        max_bins=128,\n        max_depth=31,\n        max_leaf_nodes=185,\n        random_state=2021\n    )\n    hgbc_model.fit(\n        hgbc_x_train,\n        y_train,\n    )\n\n    train_oof_preds = hgbc_model.predict(hgbc_x_valid)\n    test_oof_preds = hgbc_model.predict(test[hgbc_features])\n    hgbc_train_preds[test_index] = train_oof_preds\n    hgbc_test_preds += test_oof_preds / n_folds\n    print(\": hgbc - RMSE Score = {}\".format(rmse(y_valid, train_oof_preds)))\n    \n    \nprint(\"--> Overall metrics\")\nprint(\": XGB - rmse = {}\".format(rmse(target, xgb_train_preds)))\nprint(\": LGB - rmse = {}\".format(rmse(target, lgb_train_preds)))\nprint(\": CB - rmse = {}\".format(rmse(target, cb_train_preds)))\n# print(\": Ridge - ROC AUC Score = {}\".format(roc_auc_score(target, ridge_train_preds, average=\"micro\")))\n# print(\": SGD - ROC AUC Score = {}\".format(roc_auc_score(target, sgd_train_preds, average=\"micro\")))\nprint(\": HGBC - rmse= {}\".format(rmse(target, hgbc_train_preds)))\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:08:16.937749Z","iopub.execute_input":"2021-09-01T17:08:16.938502Z","iopub.status.idle":"2021-09-01T17:35:39.161708Z","shell.execute_reply.started":"2021-09-01T17:08:16.938449Z","shell.execute_reply":"2021-09-01T17:35:39.160325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfrom scipy.special import expit\nfrom sklearn.calibration import CalibratedClassifierCV\n\nrandom_state = 2021\nn_folds = 10\n# k_fold =KFold(n_splits=n_folds, random_state=random_state, shuffle=True)\n\nl1_train = pandas.DataFrame(data={\n    \"xgb\": xgb_train_preds.tolist(),\n    \"lgb\": lgb_train_preds.tolist(),\n    \"cb\": cb_train_preds.tolist(),\n#     \"ridge\": ridge_train_preds.tolist(),\n#     \"sgd\": sgd_train_preds.tolist(),\n    \"hgbc\": hgbc_train_preds.tolist(),\n    \"target\": target.tolist()\n})\nl1_test = pandas.DataFrame(data={\n    \"xgb\": xgb_test_preds.tolist(),\n    \"lgb\": lgb_test_preds.tolist(),\n    \"cb\": cb_test_preds.tolist(),\n#     \"sgd\": sgd_test_preds.tolist(),\n#     \"ridge\": ridge_test_preds.tolist(),    \n    \"hgbc\": hgbc_test_preds.tolist(),\n})\n\ntrain_preds = np.zeros(len(l1_train.index), )\ntest_preds1 = np.zeros(len(l1_test.index), )\nfeatures = [\"xgb\", \"lgb\",\"cb\", \"hgbc\"]\nx_train = pandas.DataFrame(l1_test[features])\ntest_preds1=(x_train[\"xgb\"]+x_train[\"lgb\"]+x_train[\"cb\"]+x_train[\"hgbc\"])/4\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:35:39.165127Z","iopub.execute_input":"2021-09-01T17:35:39.166291Z","iopub.status.idle":"2021-09-01T17:35:39.357129Z","shell.execute_reply.started":"2021-09-01T17:35:39.166235Z","shell.execute_reply":"2021-09-01T17:35:39.355974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission[\"Y2\"] = test_preds1.tolist()\nsubmission\nsubmission.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:35:39.358809Z","iopub.execute_input":"2021-09-01T17:35:39.359303Z","iopub.status.idle":"2021-09-01T17:35:39.754743Z","shell.execute_reply.started":"2021-09-01T17:35:39.359256Z","shell.execute_reply":"2021-09-01T17:35:39.753585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras.layers import Input,Dense,Dropout\nfrom tensorflow.keras import Model\nfrom  tensorflow.keras.regularizers import l2\nimport tensorflow as tf\nfrom numpy.random import seed\nseed(42)\nimport tensorflow as tf\ntf.random.set_seed(42)\nfrom tensorflow import keras\nimport numpy as np\nfrom keras import backend as K\n\n\ndef root_mean_squared_error(y_true, y_pred):\n         return K.sqrt(K.mean(K.square( (y_true - y_pred) )))\n    \n    \nfrom keras.backend import sigmoid\ndef swish(x, beta = 1):\n    return (x * sigmoid(beta * x))\nfrom keras.utils.generic_utils import get_custom_objects\nfrom keras.layers import Activation\nget_custom_objects().update({'swish': Activation(swish)})","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:35:39.757119Z","iopub.execute_input":"2021-09-01T17:35:39.757791Z","iopub.status.idle":"2021-09-01T17:35:45.708500Z","shell.execute_reply.started":"2021-09-01T17:35:39.757733Z","shell.execute_reply":"2021-09-01T17:35:45.707307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"features = [col for col in train.columns if col not in {\"Y1_mean\",\"Y2_mean\",\"UNIQUE_IDENTIFIER_\",\"SEQUENCE_NO_size\"}]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:35:45.711578Z","iopub.execute_input":"2021-09-01T17:35:45.712330Z","iopub.status.idle":"2021-09-01T17:35:45.719361Z","shell.execute_reply.started":"2021-09-01T17:35:45.712249Z","shell.execute_reply":"2021-09-01T17:35:45.717404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **DAE is Used to add diversity to the model**","metadata":{}},{"cell_type":"code","source":"def get_DAE():\n    # denoising autoencoder\n    inputs = Input((266,))\n    x = Dense(500, activation='swish')(inputs) # 1500 original\n    x = Dense(500, activation='swish', name=\"feature\")(x) # 1500 original\n    x = Dense(500, activation='swish')(x) # 1500 original\n    outputs = Dense(266, activation='linear')(x)\n    model = Model(inputs=inputs, outputs=outputs)\n    model.compile(optimizer='adam', loss=root_mean_squared_error)\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:36:35.324487Z","iopub.execute_input":"2021-09-01T17:36:35.324890Z","iopub.status.idle":"2021-09-01T17:36:35.333498Z","shell.execute_reply.started":"2021-09-01T17:36:35.324856Z","shell.execute_reply":"2021-09-01T17:36:35.332187Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"alldata = pd.concat([train[features],test[features]],axis=0)\nprint(alldata.shape)\nautoencoder = get_DAE()\nautoencoder.fit(alldata[features], alldata[features],\n                    epochs=50,\n                    batch_size=256,\n                    shuffle=True\n                    )","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:36:38.133298Z","iopub.execute_input":"2021-09-01T17:36:38.133766Z","iopub.status.idle":"2021-09-01T17:38:48.091133Z","shell.execute_reply.started":"2021-09-01T17:36:38.133718Z","shell.execute_reply":"2021-09-01T17:38:48.089533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_denoised = test.copy()\ntest[features] = autoencoder.predict(test_denoised[features])\ntrain_denoised = train.copy()\ntrain[features] = autoencoder.predict(train_denoised[features])","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:40:44.115110Z","iopub.execute_input":"2021-09-01T17:40:44.115612Z","iopub.status.idle":"2021-09-01T17:41:12.102957Z","shell.execute_reply.started":"2021-09-01T17:40:44.115576Z","shell.execute_reply":"2021-09-01T17:41:12.101607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nfrom sklearn.model_selection import KFold\nfolds=[]\nkfold = KFold(n_splits = 5, random_state = 2021, shuffle = True)\nfor fold, (trn_ind, val_ind) in enumerate(kfold.split(train)):\n    print(trn_ind,val_ind)\n    folds.append([trn_ind,val_ind])\n\n","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:43:20.775429Z","iopub.execute_input":"2021-09-01T17:43:20.775839Z","iopub.status.idle":"2021-09-01T17:43:20.803431Z","shell.execute_reply.started":"2021-09-01T17:43:20.775803Z","shell.execute_reply":"2021-09-01T17:43:20.802041Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target=['Y1_mean','Y2_mean']","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:43:22.051293Z","iopub.execute_input":"2021-09-01T17:43:22.051719Z","iopub.status.idle":"2021-09-01T17:43:22.057316Z","shell.execute_reply.started":"2021-09-01T17:43:22.051684Z","shell.execute_reply":"2021-09-01T17:43:22.055853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# from sklearn.model_selection import KFold\nimport lightgbm as lgb\n\nseed0=2021\nparams0 = {\n    'objective': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': -1,\n    'max_bin':100,\n    'min_data_in_leaf':500,\n    'learning_rate': 0.05,\n    'subsample': 0.72,\n    'subsample_freq': 4,\n    'feature_fraction': 0.5,\n    'lambda_l1': 0.5,\n    'lambda_l2': 1.0,\n    'categorical_column':[0],\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'n_jobs':-1,\n    'verbose': -1}\nseed1=42\nparams1 = {\n        'learning_rate': 0.1,        \n        'lambda_l1': 2,\n        'lambda_l2': 7,\n        'num_leaves': 800,\n        'min_sum_hessian_in_leaf': 20,\n        'feature_fraction': 0.8,\n        'feature_fraction_bynode': 0.8,\n        'bagging_fraction': 0.9,\n        'bagging_freq': 42,\n        'min_data_in_leaf': 700,\n        'max_depth': 4,\n        'categorical_column':[0],\n        'seed': seed1,\n        'feature_fraction_seed': seed1,\n        'bagging_seed': seed1,\n        'drop_seed': seed1,\n        'data_random_seed': seed1,\n        'objective': 'rmse',\n        'boosting': 'gbdt',\n        'verbosity': -1,\n        'n_jobs':-1,\n    }\n# Function to early stop with root mean squared percentage error\ndef rmspe(y_true, y_pred):\n    return np.sqrt(np.mean(np.square((y_true - y_pred))))\n\ndef feval_rmspe(y_pred, lgb_train):\n    y_true = lgb_train.get_label()\n    return 'RMSPE', rmspe(y_true, y_pred), False\n# X=train[features]\n\ndef train_and_evaluate_lgb(train, test, params, features):\n    # Hyperparammeters (just basic)\n    y = train[target[0]]\n    X=train[features]\n    # Create out of folds array\n    oof_predictions = np.zeros(train.shape[0])\n    # Create test array to store predictions\n    test_predictions = np.zeros(test.shape[0])\n    # Create a KFold object\n    kfold = KFold(n_splits = 5, random_state = 2021, shuffle = True)\n    # Iterate through each fold\n    fold=0\n    for trn_ind,val_ind in folds:\n        print(f'Training fold {fold + 1}')\n        fold+=1\n        x_train, x_val = X.iloc[trn_ind], X.iloc[val_ind]\n        y_train, y_val = y.iloc[trn_ind], y.iloc[val_ind]\n        # Root mean squared percentage error weights\n        train_dataset = lgb.Dataset(x_train[features], y_train)\n        val_dataset = lgb.Dataset(x_val[features], y_val)\n        model = lgb.train(params = params,\n                          num_boost_round=2000,\n                          train_set = train_dataset, \n                          valid_sets = [train_dataset, val_dataset], \n                          verbose_eval = 250,\n                          early_stopping_rounds=250,\n                          feval = feval_rmspe)\n        # Add predictions to the out of folds array\n        oof_predictions[val_ind] = model.predict(x_val[features])\n        # Predict the test set\n        test_predictions += model.predict(test[features]) / 5\n    rmspe_score = rmspe(y, oof_predictions)\n    print(f'Our out of folds RMSPE is {rmspe_score}')\n    lgb.plot_importance(model,max_num_features=20)\n    # Return test predictions\n    return test_predictions\n# Traing and evaluate\nfeatures = [col for col in train.columns if col not in {\"Y1_mean\",\"Y2_mean\",\"UNIQUE_IDENTIFIER_\"}]\n#features = [col for col in features if col not in cols_drop]\npredictions_lgb= train_and_evaluate_lgb(train_denoised, test_denoised, params0, features)\ntest_denoised[target[0]] = predictions_lgb\n\n# cont_features =[col for col in train.columns if col not in {\"Y1_mean\",\"Y2_mean\",\"UNIQUE_IDENTIFIER_\",\"STATUS_CHECK_mean\",\"SEQUENCE_NO_size\"}]\n# cat_features = [\"STATUS_CHECK_mean\",\"SEQUENCE_NO_size\"]","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:43:23.594152Z","iopub.execute_input":"2021-09-01T17:43:23.594582Z","iopub.status.idle":"2021-09-01T17:48:48.552750Z","shell.execute_reply.started":"2021-09-01T17:43:23.594544Z","shell.execute_reply":"2021-09-01T17:48:48.551217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import KFold\nimport lightgbm as lgb\n\nseed0=2021\nparams0 = {\n    'objective': 'rmse',\n    'boosting_type': 'gbdt',\n    'max_depth': -1,\n    'max_bin':100,\n    'min_data_in_leaf':500,\n    'learning_rate': 0.05,\n    'subsample': 0.72,\n    'subsample_freq': 4,\n    'feature_fraction': 0.5,\n    'lambda_l1': 0.5,\n    'lambda_l2': 1.0,\n    'categorical_column':[0],\n    'seed':seed0,\n    'feature_fraction_seed': seed0,\n    'bagging_seed': seed0,\n    'drop_seed': seed0,\n    'data_random_seed': seed0,\n    'n_jobs':-1,\n    'verbose': -1}\nseed1=42\nparams1 = {\n        'learning_rate': 0.1,        \n        'lambda_l1': 2,\n        'lambda_l2': 7,\n        'num_leaves': 800,\n        'min_sum_hessian_in_leaf': 20,\n        'feature_fraction': 0.8,\n        'feature_fraction_bynode': 0.8,\n        'bagging_fraction': 0.9,\n        'bagging_freq': 42,\n        'min_data_in_leaf': 700,\n        'max_depth': 4,\n        'categorical_column':[0],\n        'seed': seed1,\n        'feature_fraction_seed': seed1,\n        'bagging_seed': seed1,\n        'drop_seed': seed1,\n        'data_random_seed': seed1,\n        'objective': 'rmse',\n        'boosting': 'gbdt',\n        'verbosity': -1,\n        'n_jobs':-1,\n    }\n# Function to early stop with root mean squared percentage error\ndef rmspe(y_true, y_pred):\n    return np.sqrt(np.mean(np.square((y_true - y_pred))))\n\ndef feval_rmspe(y_pred, lgb_train):\n    y_true = lgb_train.get_label()\n    return 'RMSPE', rmspe(y_true, y_pred), False\nX=train_denoised[features]\n\ndef train_and_evaluate_lgb(train, test, params, features):\n    # Hyperparammeters (just basic)\n    y = train[target[1]]\n    X=train[features]\n    # Create out of folds array\n    oof_predictions = np.zeros(train.shape[0])\n    # Create test array to store predictions\n    test_predictions = np.zeros(test.shape[0])\n    # Create a KFold object\n    kfold = KFold(n_splits = 5, random_state = 2021, shuffle = True)\n    # Iterate through each fold\n    fold=0\n    for trn_ind,val_ind in folds:\n        print(f'Training fold {fold + 1}')\n        fold+=1\n        x_train, x_val = X.iloc[trn_ind], X.iloc[val_ind]\n        y_train, y_val = y.iloc[trn_ind], y.iloc[val_ind]\n        # Root mean squared percentage error weights\n        train_dataset = lgb.Dataset(x_train[features], y_train)\n        val_dataset = lgb.Dataset(x_val[features], y_val)\n        model = lgb.train(params = params,\n                          num_boost_round=2000,\n                          train_set = train_dataset, \n                          valid_sets = [train_dataset, val_dataset], \n                          verbose_eval = 250,\n                          early_stopping_rounds=500,\n                          feval = feval_rmspe)\n        # Add predictions to the out of folds array\n        oof_predictions[val_ind] = model.predict(x_val[features])\n        # Predict the test set\n        test_predictions += model.predict(test[features]) / 5\n    rmspe_score = rmspe(y, oof_predictions)\n    print(f'Our out of folds RMSPE is {rmspe_score}')\n    lgb.plot_importance(model,max_num_features=20)\n    # Return test predictions\n    return test_predictions\n# Traing and evaluate\nfeatures = [col for col in train.columns if col not in {\"Y1_mean\",\"Y2_mean\",\"UNIQUE_IDENTIFIER_\"}]\n#features = [col for col in features if col not in cols_drop]\npredictions_lgb= train_and_evaluate_lgb(train_denoised, test_denoised, params0, features)\ntest_denoised[target[1]] = predictions_lgb","metadata":{"execution":{"iopub.status.busy":"2021-09-01T17:48:56.987968Z","iopub.execute_input":"2021-09-01T17:48:56.988453Z","iopub.status.idle":"2021-09-01T17:51:57.093427Z","shell.execute_reply.started":"2021-09-01T17:48:56.988418Z","shell.execute_reply":"2021-09-01T17:51:57.091893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission1 = pandas.read_csv(\"../input/beyond-analysis/sample_submission_random.csv\")\nsubmission1['Y1']=0.50*submission['Y1']+0.50*test_denoised['Y1_mean']\nsubmission1['Y2']=0.50*submission['Y2']+0.50*test_denoised['Y2_mean']","metadata":{"execution":{"iopub.status.busy":"2021-09-01T18:00:48.373295Z","iopub.execute_input":"2021-09-01T18:00:48.373906Z","iopub.status.idle":"2021-09-01T18:00:48.467028Z","shell.execute_reply.started":"2021-09-01T18:00:48.373858Z","shell.execute_reply":"2021-09-01T18:00:48.465856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission1.to_csv(\"submission1.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-01T18:00:50.231571Z","iopub.execute_input":"2021-09-01T18:00:50.231991Z","iopub.status.idle":"2021-09-01T18:00:50.712794Z","shell.execute_reply.started":"2021-09-01T18:00:50.231957Z","shell.execute_reply":"2021-09-01T18:00:50.711225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **CV OBSERVATIONS**\n*  cv was correlating with public leaderboard.\n* i was able to get best rmse for y1 round of  6.70 and for y2 =119.31","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}