{"cells":[{"metadata":{},"cell_type":"markdown","source":">**Import the libraies**"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-output":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sn\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom sklearn.metrics import make_scorer,mean_squared_error\nfrom sklearn.model_selection import cross_val_score\nimport xgboost as xgb\nfrom catboost import Pool, CatBoostRegressor\nimport random\nrandom.seed(42)\nimport os\nprint(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Pre-processing**"},{"metadata":{},"cell_type":"markdown","source":"\n> Load the data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"\npot_energy=pd.read_csv('../input/potential_energy.csv')\nmulliken_charges=pd.read_csv('../input/mulliken_charges.csv')\ntrain_df=pd.read_csv('../input/train.csv')\nscalar_coupling_cont=pd.read_csv('../input/scalar_coupling_contributions.csv')\ntest_df=pd.read_csv('../input/test.csv')\nmagnetic_shield_tensor=pd.read_csv('../input/magnetic_shielding_tensors.csv')\ndipole_moment=pd.read_csv('../input/dipole_moments.csv')\nstructures=pd.read_csv('../input/structures.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n> Let's see the shapes of our datasets"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"print('Shape of potential energy dataset:',pot_energy.shape)\nprint('Shape of mulliken_charges dataset:',mulliken_charges.shape)\nprint('Shape of train dataset:',train_df.shape)\nprint('Shape of scalar coupling contributions dataset:',scalar_coupling_cont.shape)\nprint('Shape of test dataset:',test_df.shape)\nprint('Shape of magnetic shielding tensors dataset:',magnetic_shield_tensor.shape)\nprint('Shape of dipole moments dataset:',dipole_moment.shape)\nprint('Shape of structures dataset:',structures.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Let's explore our datasetes"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#data types in dataset\nprint('Data Types:\\n',pot_energy.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(pot_energy.describe(),3))\n#Top few rows of dataset\npot_energy.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> molecule_name - Name of the molecule,  \n>potential_energy - Potential energy of the molecule"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#data types in dataset\nprint('Data Types:\\n',mulliken_charges.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(mulliken_charges.describe(),3))\n#Top few rows of dataset\nmulliken_charges.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> atom_index - Index of the atom in molecule,  \n> mulliken_charge- Mulliken charge of the atom"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#data types in dataset\nprint('Data Types:\\n',train_df.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(train_df.describe(),3))\n#Top few rows of dataset\ntrain_df.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> atom_index_1 - atom indices of the atom-pair creating coupling,  \n> type - scalar coupling type,  \n> scalar_coupling_constant- scalar coupling constant that we want to predict (target variable)"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#data types in dataset\nprint('Data Types:\\n',scalar_coupling_cont.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(scalar_coupling_cont.describe(),3))\n#Top few rows of dataset\nscalar_coupling_cont.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> fc  - Fermi contact contribution,  \n> sc  - Spin-dipolar contribution,  \n> pso - Paramagnetic spin-orbit contribution,  \n> dso - Diemagnetic spin-orbit contribution"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#data types in dataset\nprint('Data Types:\\n',test_df.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(test_df.describe(),3))\n#Top few rows of dataset\ntest_df.head(6)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"\n#data types in dataset\nprint('Data Types:\\n',magnetic_shield_tensor.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(magnetic_shield_tensor.describe(),3))\n#Top few rows of dataset\nmagnetic_shield_tensor.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> XX, YX, ZX, XY, YY, ZY, XZ, YZ, ZZ- Elements of tensor/ matrix"},{"metadata":{},"cell_type":"markdown","source":"> X, Y, Z - X, Y, & Z componets of dipole moments"},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#data types in dataset\nprint('Data Types:\\n',structures.dtypes)\n#Descriptive statistics\nprint('Descriptive statistics:\\n',np.round(structures.describe(),3))\n#Top few rows of dataset\nstructures.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> x, y, & z - Cartesian co-oridnates of the atom"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Map the atom structure data into train & test datasets\n\ndef map_atom_data(df,atom_idx):\n    df=pd.merge(df,structures,how='left',\n               left_on=['molecule_name',f'atom_index_{atom_idx}'],\n               right_on=['molecule_name','atom_index'])\n    df=df.drop('atom_index',axis=1)\n    df=df.rename(columns={'atom':f'atom_{atom_idx}',\n                         'x':f'x_{atom_idx}',\n                         'y':f'y_{atom_idx}',\n                         'z':f'z_{atom_idx}'})\n    return df\n#train dataset\ntrain_df=map_atom_data(train_df,0)\ntrain_df=map_atom_data(train_df,1)\n#test dataset\ntest_df=map_atom_data(test_df,0)\ntest_df=map_atom_data(test_df,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Engineer a single feature: distance vector between atoms\n# for train dataset\ntrain_m_0=train_df[['x_0','y_0','z_0']].values\ntrain_m_1=train_df[['x_1','y_1','z_1']].values\n#for test dataset\ntest_m_0=test_df[['x_0','y_0','z_0']].values\ntest_m_1=test_df[['x_0','y_0','z_0']].values\n\n#distance vector between atoms for train dataset\ntrain_df['dist_vector']=np.linalg.norm(train_m_0-train_m_1,axis=1)\ntrain_df['dist_X']=(train_df['x_0']-train_df['x_1'])**2\ntrain_df['dist_Y']=(train_df['y_0']-train_df['y_1'])**2\ntrain_df['dist_Z']=(train_df['z_0']-train_df['z_1'])**2\n\n#distance vector between atoms for test dataset\ntest_df['dist_vector']=np.linalg.norm(test_m_0-test_m_1,axis=1)\ntest_df['dist_X']=(test_df['x_0']-test_df['x_1'])**2\ntest_df['dist_Y']=(test_df['y_0']-test_df['y_1'])**2\ntest_df['dist_Z']=(test_df['z_0']-test_df['z_1'])**2\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> > **References:-**     \n> 1. https://www.kaggle.com/abhishek/competition-metric,   \n> 2. https://www.kaggle.com/seriousran/just-speed-up-calculate-distance-from-benchmark,  \n> 3. https://www.kaggle.com/inversion/atomic-distance-benchmark,    \n> 4. https://www.kaggle.com/robikscube/exploring-molecular-properties-data"},{"metadata":{},"cell_type":"markdown","source":"> Now let's check our train & test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df['type_0']=train_df['type'].apply(lambda x:x)\n#test_df['type_0']=test_df['type'].apply(lambda x : x)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"train_df=train_df.drop(columns=['molecule_name'],axis=1)\ndisplay(train_df.head(6))","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"test_df=test_df.drop(columns=['molecule_name'],axis=1)\ndisplay(test_df.head(10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Attributes like **atom_0**, & **atom_1** requires type converson from **obejcet** to **categorical** for proper understanding"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Type casting\n\n# for train data\n#train_df['type_0']=train_df.type_0.astype('category')\ntrain_df['type']=train_df.type.astype('category')\ntrain_df['atom_0']=train_df.atom_0.astype('category')\ntrain_df['atom_1']=train_df.atom_1.astype('category')\n\n#for test data\n#test_df['type_0']=test_df.type_0.astype('category')\ntest_df['type']=test_df.type.astype('category')\ntest_df['atom_0']=test_df.atom_0.astype('category')\ntest_df['atom_1']=test_df.atom_1.astype('category')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Histograms for Data Visualization**"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['scalar_coupling_constant'])\nplt.ylabel('No of times')\nplt.xlabel('scalar copling constant')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_vector'])\nplt.ylabel('No of times')\nplt.xlabel('Distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_X'])\nplt.ylabel('No of times')\nplt.xlabel('X distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_Y'])\nplt.ylabel('No of times')\nplt.xlabel('Y distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_Z'])\nplt.ylabel('No of times')\nplt.xlabel('Z distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":">** Box plots to detect outliers**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train=train_df[['scalar_coupling_constant','dist_vector','dist_X','dist_Y','dist_Z']]\nplt.figure(figsize=(20,16))\nsn.boxplot(data=train)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test=test_df[['dist_vector','dist_X','dist_Y','dist_Z']]\nplt.figure(figsize=(15,8))\nsn.boxplot(data=test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Replace & impute the outliers"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dataframe for outliers\ntrainDf=pd.DataFrame(train,columns=['scalar_coupling_constant','dist_X','dist_Y','dist_Z'])\ntestDf=pd.DataFrame(test,columns=['dist_X','dist_Y','dist_Z'])\ntrain_labels=['scalar_coupling_constant','dist_X','dist_Y','dist_Z']\ntest_labels=['dist_X','dist_Y','dist_Z']\nfor i in train_labels:\n    # Divide the data into 75% quantile & 25% quantile                               \n    q75,q25=np.percentile(trainDf.loc[:,i],[75,25]) \n    #Inter quantile range\n    iqr=q75-q25\n    #Inner fence\n    min=q25-(iqr*1.5)\n    #outer fence\n    max=q75+(iqr*1.5)\n    #Replace outliers with NA\n    trainDf.loc[trainDf.loc[:,i]<min,:i]=np.NaN\n    trainDf.loc[trainDf.loc[:,i]>max,:i]=np.NaN\n        \n        \nfor j in test_labels:\n    # Divide the data into 75% quantile & 25% quantile                               \n    q75,q25=np.percentile(testDf.loc[:,j],[75,25]) \n    #Inter quantile range\n    iqr=q75-q25\n    #Inner fence\n    min=q25-(iqr*1.5)\n    #outer fence\n    max=q75+(iqr*1.5)\n    #Replace outliers with NA\n    testDf.loc[testDf.loc[:,j]<min,:j]=np.NaN\n    testDf.loc[testDf.loc[:,j]>max,:j]=np.NaN\n                                  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(trainDf.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(testDf.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Imputing the outliers by mean imputation\n#For train data\ntrainDf['scalar_coupling_constant']=trainDf['scalar_coupling_constant'].fillna(trainDf['scalar_coupling_constant'].mean())\ntrainDf['dist_X']=trainDf['dist_X'].fillna(trainDf['dist_X'].mean())\ntrainDf['dist_Y']=trainDf['dist_Y'].fillna(trainDf['dist_Y'].mean())\ntrainDf['dist_Z']=trainDf['dist_Z'].fillna(trainDf['dist_Z'].mean())\n#For test data\ntestDf['dist_X']=testDf['dist_X'].fillna(testDf['dist_X'].mean())\ntestDf['dist_Y']=testDf['dist_Y'].fillna(testDf['dist_Y'].mean())\ntestDf['dist_Z']=testDf['dist_Z'].fillna(testDf['dist_Z'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(trainDf.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(testDf.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Replace the features in original dataset with imputed data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=train_df.drop(columns=['scalar_coupling_constant','dist_X','dist_Y','dist_Z'],axis=1)\ntest_df=test_df.drop(columns=['dist_X','dist_Y','dist_Z'],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#For tran dataset\ntrain_df['scalar_coupling_constant']=trainDf['scalar_coupling_constant']\ntrain_df['dist_X']=trainDf['dist_X']\ntrain_df['dist_Y']=trainDf['dist_Y']\ntrain_df['dist_Z']=trainDf['dist_Z']\n#For test dataset\ntest_df['dist_X']=testDf['dist_X']\ntest_df['dist_Y']=testDf['dist_Y']\ntest_df['dist_Z']=testDf['dist_Z']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Histogram plots after outliers imputation**"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['scalar_coupling_constant'])\nplt.ylabel('No of times')\nplt.xlabel('scalar coupling constant')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_vector'])\nplt.ylabel('No of times')\nplt.xlabel('distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_X'])\nplt.ylabel('No of times')\nplt.xlabel('X distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_Y'])\nplt.ylabel('No of times')\nplt.xlabel('Y distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(train_df['dist_Z'])\nplt.ylabel('No of times')\nplt.xlabel('Z distance vector')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Feature selection**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Threshold for removing correalated values\n#threshold=0.95\n#Absolute value correlation matrix\n#corr_matrix=train_df.corr().abs()\n\n#Gettng the upper traingle of correlations\n#upper=corr_matrix.where(np.triu(np.ones(corr_matrix.shape),k=1).astype(np.bool))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#select columns with correlations above threshold\n#to_drop=[column for column in upper.columns if any(upper[column]>threshold)]\n#print('There are are %d columns to remove.'%(len(to_drop)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df=train_df.drop(columns=to_drop)\n#test_df=test_df.drop(columns=to_drop)\n#print('Training data shape',train_df.shape)\n#print('Testing data shape',test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Attributes=['atom_index_0','atom_index_1','type','x_0','y_0','z_0','atom_0',\n            'atom_1','x_1','y_1','z_1','dist_vector','dist_X','dist_Y','dist_Z']\n#categorical attributes\ncat_attributes=['type','atom_0','atom_1']\ntarget_label=['scalar_coupling_constant']\n# split the data into X_train,X_test,& y_target\n\nX_train=train_df[Attributes]\nX_test=test_df[Attributes]\ny_target=train_df[target_label]\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n#Transform categorical variables\n#X_train=pd.get_dummies(X_train,columns=cat_attributes)\n#X_test=pd.get_dummies(X_test,columns=cat_attributes)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"print(X_train.shape,X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"display(y_target.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"#Enocode the categorigal variables\nfrom sklearn.preprocessing import LabelEncoder\nfor f in ['type','atom_index_0','atom_index_1','atom_0','atom_1']:\n    #if f in good_columns:\n        lbl=LabelEncoder()\n        lbl.fit(list(X_train[f].values)+list(X_test[f].values))\n        X_train[f]=lbl.transform(list(X_train[f].values))\n        X_test[f]=lbl.transform(list(X_test[f].values))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Let's look at our transformed data"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nX_train.head(6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.head(6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Linear Regression model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfrom sklearn import linear_model\n#linear_reg=linear_model.LinearRegression()\n#n_folds=3\n#Cross validation\n#lin_reg_score=cross_val_score(linear_reg,X_train,y_target,\n                          #scoring=make_scorer(mean_squared_error),\n                          #cv=n_folds)\n#lin_score=sum(lin_reg_score)/n_folds\n#print('rgf_score:',lin_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lr_model=linear_reg.fit(X_train,y_target)\n#score=np.round(lr_model.score(X_train,y_target),3)\n#print('Accuracy of trained model:',score)\n#model_coeff=np.round(lr_model.coef_,3)\n#print('Model coefficients:',model_coeff)\n#model_intercept=np.round(lr_model.intercept_,3)\n#print('Model intercept value:',model_intercept)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model prediction\n#from sklearn.metrics import r2_score\n#y_pred=lr_model.predict(X_test)\n#SCC=pd.read_csv('../input/sample_submission.csv')\n#SCC['scalar_coupling_constant']= y_pred\n#SCC.to_csv('Linear_Regression_model.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Lasso Regression model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\n#from sklearn import linear_model\n#lasso=linear_model.Lasso(alpha=0.001)\n#n_folds=10\n#Cross validation\n#lasso_score=cross_val_score(lasso,X_train,y_target,\n                          #scoring=make_scorer(mean_squared_error),\n                          #cv=n_folds)\n#lasso_score=sum(lasso_score)/n_folds\n#print('lasso_score:',lasso_score)\n#print(lasso)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#lasso_model=lasso.fit(X_train,y_target)\n#score=np.round(lasso_model.score(X_train,y_target),3)\n#print('Accuracy of trained model:',score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model prediction\n#y_pred=lasso_model.predict(X_test)\n#SCC=pd.read_csv('../input/sample_submission.csv')\n#SCC['scalar_coupling_constant']= y_pred\n#SCC.to_csv('Lasso_Regression_model.csv',index=False)\n#y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from sklearn import linear_model\n#linear_reg=linear_model.ElasticNet(alpha=0.008)\n#ElasticNet_model=linear_reg.fit(X_train,y_target)\n#score=np.round(ElasticNet_model.score(X_train,y_target),3)\n#print('Accuracy of trained model:',score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#model prediction\n#y_pred=ElasticNet_model.predict(X_test)\n#SCC=pd.read_csv('../input/sample_submission.csv')\n#SCC['scalar_coupling_constant']= y_pred\n#SCC.to_csv('ElasticNet_Regression_model.csv',index=False)\n#y_pred","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\n**Light Gradient Boostng Machine**"},{"metadata":{},"cell_type":"markdown","source":"> Hyperopt for Hyperparameter tuning (Bayesian optimization)"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Define hyper space\n#from hyperopt import fmin,hp,tpe,Trials,space_eval,STATUS_OK,STATUS_RUNNING\n#hyper_space={'objective':'regression',\n            # 'metric':'mape',\n             #'boosting':'gbdt',\n             #'n_estimators':hp.choice('n_estimators',[100,250,450,600,850,1000,2000,3000,4000,5000]),\n            # 'max_depth':hp.choice('max_depth',[5,10,15,20,25,30,35]),\n             #'num_leaves':hp.choice('num_leaves',[45,60,95,125,145,200]),\n             #'subsample':hp.choice('subsample',[.1,.2,.3,.4,0.5,0.6,0.7,0.8,0.9,1]),\n             #'colsample_bytree': hp.choice('colsample_bytree',[.1,.2,.3,.4,.5,0.6,0.7,0.8,0.9,1.0]),\n             #'learning_rate': hp.choice('learning_rate',[0.1,0.2,0.3,0.35,0.4,0.5]),\n             #'reg_lambda': hp.choice('reg_lambda',[.1,.2,.3,.4,.5,.6,.7,.8,.9,1.0]),\n            # 'reg_alpha': hp.choice('reg_alpha',[.1,.2,.3,.4,.5,.6,.7,.8,.9,1.0]),\n            # 'min_child_samples':hp.choice('min_child_samples',[3,6,8,12,15])\n           # }\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Defining the metric to score our optimizer\n#def metric(df,pred):\n    #df['diff']=(df['scalar_coupling_constant']-pred).abs()\n    #return np.log(df.groupby([['type']])['diff'].mean().map(lambda x:max(x,1e-9))).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Split the train data into train(70%) & valid dataset(30%)\nfrom sklearn.model_selection import train_test_split\nX_train,X_valid,y_train,y_valid=train_test_split(X_train,y_target,\n                                               test_size=0.2, random_state=42)\nprint(X_train.shape,X_valid.shape)\nprint(y_train.shape,y_valid.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from lightgbm import LGBMRegressor\n#import lightgbm as lgb\n#Create a function for Hyper parameters optimization\n#train_data=lgb.Dataset(X_train,label=y_train)\n#valid_data=lgb.Dataset(X_valid,label=y_valid)\n#def evaluate_metric(params):\n    #lgbm_model=lgb.train(params,train_data,500,\n                             # valid_sets=[train_data,valid_data],\n                            # early_stopping_rounds=20,verbose_eval=500)\n   # pred=lgbm_model.predict(X_valid)\n   # score=metric(df_valid,pred)\n   # print(score)\n   # return {\n        #'loss':score,\n        #'status':STATUS_OK,\n       # 'staus running':STATUS_RUNNING\n    #}\n    ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Initalzing the optimizer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#from functools import partial\n#Trial\n#trials=Trials()\n#set algorithm parameters\n#algo=partial(tpe.suggest,n_startup_jobs=-1)\n#set the no.of evaluations\n#Max_evals=20\n#Fit the Tree parzen estimator\n#best_vals=fmin(evaluate_metric,space=hyper_space,verbose=1,\n             # algo=algo,max_evals=Max_evals,trials=trials)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print the best parameters\n#best_params=space_eval(hyper_space,best_vals)\n#print('Best Parameters from Hyperopt:\\n'+str(best_params))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#params={'objective':'regression',\n       # 'boosting':'dart',\n        #'colsample_bytree':1.0,\n        #'learning_rate':0.35,\n        #'max_depth':30,\n        #'metric':'mse',\n        #'min_child_samples':120,\n        #'n_estimators':5000,\n        #'num_leaves':2000,\n        #'reg_alpha':0.005,\n       #}\n\n#lgb_model=lgb.train(params,train_data,valid_sets=[train_data,valid_data],verbose_eval=100,\n                 #  early_stopping_rounds=100)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y_pred=lgb_model.predict(X_test)\n#SCC=pd.read_csv('../input/sample_submission.csv')\n#SCC['scalar_coupling_constant']= y_pred\n#SCC.to_csv('LGBM_dart_model.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **XGBOOST**"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#training dataset\nxg_train=xgb.DMatrix(data=X_train,label=y_train)\nxg_valid=xgb.DMatrix(data=X_valid,label=y_valid)\nxg_test=xgb.DMatrix(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Setting parameters\nxgb_params={'booster':'gbtree',\n            'eval_metric':'rmse',\n            'eta':0.5,\n            'gamma':20,\n            'max_depth':30,\n           }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nnum_rounds=1000\nevallist=[(xg_train,'train'),(xg_valid,'eval')]\nxgb=xgb.train(xgb_params,xg_train,num_rounds,evallist,verbose_eval=100,early_stopping_rounds=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nxgb_predict=xgb.predict(xg_test,ntree_limit=xgb.best_ntree_limit)\n\nSCC=pd.read_csv('../input/sample_submission.csv')\nSCC['scalar_coupling_constant']= xgb_predict\nSCC.to_csv('xgboost_model.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Regurized Greedy Forest (RGF) algorithm**"},{"metadata":{"trusted":false},"cell_type":"code","source":"from rgf.sklearn import RGFRegressor,FastRGFRegressor\nfrom sklearn.metrics import make_scorer,mean_squared_error\nfrom sklearn.model_selection import cross_val_score\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#%%time\n#rgf=RGFRegressor(max_leaf=500,algorithm='RGF_Sib',test_interval=100,\n               # loss='LS',verbose=False)\n#n_folds=3\n#rgf_scores = cross_val_score(rgf,\n                             #X_train,\n                            # y_target,\n                             #scoring=make_scorer(mean_squared_error,greater_is_better=False),\n                             #cv=n_folds)\n#rgf_score=sum(rgf_scores)/n_folds\n#print('rgf_score:',rgf_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#%%time\n#Tran the model\n#rgf_model=rgf.fit(X_train,y_target)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":false},"cell_type":"code","source":"#%%time\n#model prediction\n#y_pred=rgf_model.predict(X_test)\n#SCC=pd.read_csv('../input/sample_submission.csv')\n#SCC['scalar_coupling_constant']= y_pred\n#SCC.to_csv('RGF_model.csv',index=False)\n#y_pred\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Fast Regurized Greedy Forest (RGF) algorithm**"},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\n#Frgf=FastRGFRegressor( opt_algorithm='rgf',\n                     #l2=2000.0,\n                     #min_child_weight=5.0,\n                     #sparse_max_features=80000,\n                    # sparse_min_occurences=5\n                     #)\n#Frgf=FastRGFRegressor(n_estimators=1000)\n#n_folds=3\n#Frgf_scores=cross_val_score(Frgf,X_train,y_target,\n                         # scoring=make_scorer(mean_squared_error),\n                          #cv=n_folds)\n#Frgf_score=sum(Frgf_scores)/n_folds\n#print('Frgf_score:',Frgf_score)\n#print(Frgf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#Train the model\n#Frgf_model=Frgf.fit(X_train,y_target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"#model prediction\n#y_pred=Frgf_model.predict(X_test)\n#SCC=pd.read_csv('../input/sample_submission.csv')\n#SCC['scalar_coupling_constant']= y_pred\n#SCC.to_csv('FRGF_model.csv',index=False)\n#y_pred\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.4"}},"nbformat":4,"nbformat_minor":1}