{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport time\nimport gc\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dtypes = {'atom_index_0':'uint8', \n          'atom_index_1':'uint8', \n          'scalar_coupling_constant':'float32', \n          'num_C':'uint8', \n          'num_H':'uint8', \n          'num_N':'uint8', \n          'num_O':'uint8', \n          'num_F':'uint8',\n          'total_atoms':'uint8',\n          'num_bonds':'uint8', \n          'num_mol_bonds':'uint8', \n          'min_d':'float32', \n          'mean_d':'float32', \n          'max_d':'float32', \n          'space_dr':'float32', \n          'bond_dr':'float32',\n          'bond_1':'uint8', \n          'bond_2':'uint8', \n          'bond_3':'uint8', \n          'atom_0_pc':'float32', \n          'atom_end_pc':'float32',\n          'atom_2_hyb':'uint8', \n          'atom_3_hyb':'uint8', \n          'atom_end_hyb':'uint8', \n          'path_count':'uint8', \n          'atom_0_min':'float32',\n          'atom_0_mean':'float32', \n          'atom_0_max':'float32', \n          'atom_0_Cmin':'float32', \n          'atom_0_Cmean':'float32',\n          'atom_0_Cmax':'float32', \n          'atom_0_Omin':'float32', \n          'atom_0_Omean':'float32',\n          'atom_0_Omax':'float32', \n          'atom_0_Nmin':'float32', \n          'atom_0_Nmean':'float32', \n          'atom_0_Nmax':'float32',\n          'atom_0_Fmin':'float32', \n          'atom_0_Fmean':'float32', \n          'atom_0_Fmax':'float32', \n          'atom_end_min':'float32',\n          'atom_end_mean':'float32', \n          'atom_end_max':'float32', \n          'atom_end_Cmin':'float32', \n          'atom_end_Cmean':'float32',\n          'atom_end_Cmax':'float32', \n          'atom_end_Omin':'float32', \n          'atom_end_Omean':'float32',\n          'atom_end_Omax':'float32', \n          'atom_end_Nmin':'float32', \n          'atom_end_Nmean':'float32', \n          'atom_end_Nmax':'float32',\n          'atom_end_Fmin':'float32', \n          'atom_end_Fmean':'float32', \n          'atom_end_Fmax':'float32',\n          'Dmin_COM':'float32', \n          'Dmean_COM':'float32', \n          'Dmax_COM':'float32',\n          'COM_dr_0': 'float32',\n          'COM_dr_1': 'float32',\n          'bond2_angle': 'float32',\n          'bond3_angle': 'float32'\n         }","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/predmolprop-featureengineering-final/train_extend.csv\",dtype=dtypes)\ntest = pd.read_csv(\"../input/predmolprop-featureengineering-finaltest/test_extend.csv\",dtype=dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PopList = ['molecule_name', 'atom_index_0','atom_index_1','num_bonds','atom_end_type',\n           'atom_2_hyb','atom_3_hyb','atom_end_hyb','bond_1']\n\nfor col in PopList:\n    train.pop(col)\n    test.pop(col)\n    \ntrain.fillna(value ='',inplace= True)\ntest.fillna(value='',inplace=True)\n\ncoupling_types = sorted(list(train.type.unique()))\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train[train.bond2_angle!=-1].bond2_angle)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.loc[train.bond3_angle.map(lambda x: float(x)==type(''))].bond3_angle","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.bond3_angle.map(lambda x: float(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encode categorical features\nfrom sklearn.preprocessing import LabelEncoder\n\ncols = ['atom_0_type2','atom_2_type','atom_3_type','atom_end_type2']\nfor col in cols:\n    enc = LabelEncoder()\n    train[col]=enc.fit_transform(train[col]).astype(np.uint8)\n    test[col]=enc.transform(test[col]).astype(np.uint8)\ndel cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature lists\nfc_size = ['num_mol_bonds', 'min_d','mean_d', 'max_d', 'total_atoms',\n           'Dmin_COM', 'Dmean_COM', 'Dmax_COM',\n           'num_C', 'num_H', 'num_N', 'num_O', 'num_F']\n\nfc_atom_0 = ['atom_0_pc', 'atom_0_type2','COM_dr_0',\n             'atom_0_min','atom_0_mean', 'atom_0_max', \n             'atom_0_Cmin', 'atom_0_Cmean','atom_0_Cmax', \n             'atom_0_Omin', 'atom_0_Omean','atom_0_Omax', \n             'atom_0_Nmin', 'atom_0_Nmean', 'atom_0_Nmax',\n             'atom_0_Fmin', 'atom_0_Fmean', 'atom_0_Fmax']\n\nfc_atom_end = ['atom_end_pc', 'atom_end_type2', 'COM_dr_1',\n               'atom_end_min','atom_end_mean', 'atom_end_max', \n               'atom_end_Cmin', 'atom_end_Cmean','atom_end_Cmax', \n               'atom_end_Omin', 'atom_end_Omean','atom_end_Omax', \n               'atom_end_Nmin', 'atom_end_Nmean', 'atom_end_Nmax',\n               'atom_end_Fmin', 'atom_end_Fmean', 'atom_end_Fmax', ]\n\nfc_distance = ['path_count','space_dr', 'bond_dr']\n\nfc1 = set(fc_size+fc_atom_0+fc_atom_end)\n\nfc2 = set(['atom_2_type','bond_2','bond2_angle'])\nfc3 = set(['atom_2_type','atom_3_type','bond_2', 'bond_3','bond3_angle'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process Data\nfrom sklearn.model_selection import train_test_split\ndef ProcessData(df,features,test_size=0.25):\n    if test_size == 0:\n        train_Y = df.pop('scalar_coupling_constant')\n        train_type = df.pop('type')\n        df.pop('id')\n        return df.loc[:,df.columns.map(lambda x: x in features)], train_Y, train_type\n    \n    train_X, val_X, train_Y, val_Y = train_test_split(df.loc[:,df.columns.map(lambda x: x in features or x=='type')], \n                                                      df.scalar_coupling_constant,test_size=test_size,random_state=42)\n    \n    train_type = train_X.pop('type')\n    val_type = val_X.pop('type')\n    return train_X, train_Y, train_type, val_X, val_Y, val_type","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"[https://www.kaggle.com/uberkinder/efficient-metric](http://)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def CalcLMAE(y_true, y_pred, groups, floor=1e-9):\n    maes = (y_true-y_pred).abs().groupby(groups).mean()\n    return np.log(maes.map(lambda x: max(x, floor))).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from xgboost import XGBRegressor\nfrom xgboost import XGBRFRegressor\nfrom math import log\n\ndef SingleRun(df,features,test_size=0.25,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,**kwargs):\n    data = ProcessData(df,features,test_size)\n    if(test_size==0):\n        train_X,train_Y,train_type = data\n    else:\n        train_X,train_Y,train_type,val_X,val_Y,val_type = data\n    if includeType:\n        train_X=train_X.join(train_type)\n        val_X=val_X.join(val_type)\n        enc = LabelEncoder()\n        train_X.type=enc.fit_transform(train_X.type).astype(np.uint8)\n        val_X.type=enc.transform(val_X.type).astype(np.uint8)\n    \n    model = model_fn(**kwargs)\n    t1=time.time()\n    if early_stopping_rounds is None:\n        model.fit(train_X,train_Y, verbose=False)\n    else:\n        if test_size==0:\n            print('ERROR: need test data for early_stopping_rounds')\n            return -1,-1,-1\n        model.fit(train_X,train_Y, early_stopping_rounds=5, eval_set=[(val_X, val_Y)], verbose=False)\n        print('best LMAE:',log(model.best_score))\n        print('best ntree:', model.best_ntree_limit)\n    t2=time.time()\n    print('training time:',t2-t1)\n    train_predict = pd.Series(model.predict(train_X),index=train_X.index)\n    train_LMAE = CalcLMAE(train_Y,train_predict,train_type)\n    print('\\ttrain LMAE:',train_LMAE)\n    if(test_size==0):\n        val_LMAE = None\n        g = sns.FacetGrid(pd.DataFrame({'type':train_type,'scalar_coupling_constant': train_Y,'predictions':train_predict}), \n                          col=\"type\", col_order = coupling_types,sharex=False,sharey=False)\n        g.map(sns.scatterplot, \"scalar_coupling_constant\",\"predictions\")\n    else:\n        val_predict = pd.Series(model.predict(val_X),index=val_X.index)\n        val_LMAE = CalcLMAE(val_Y,val_predict,val_type)\n        print('\\tval LMAE:',val_LMAE)\n        g = sns.FacetGrid(pd.DataFrame({'type':val_type,'scalar_coupling_constant': val_Y,'predictions':val_predict}), \n                          col=\"type\", col_order = coupling_types,sharex=False,sharey=False)\n        g.map(sns.scatterplot, \"scalar_coupling_constant\",\"predictions\")\n    \n    gc.collect()\n    return model, train_LMAE, val_LMAE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First use a fairly high learning rate\ntrain_sample = train[train.type=='1JHN']\nmodel_1JHN,_,_=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=5,\n                max_depth=11, learning_rate=0.25, n_estimators=500, \n                verbosity=1, \n                objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n                n_jobs=4, \n                gamma=0, min_child_weight=1, max_delta_step=0, \n                subsample=0.8, colsample_bytree=0.2, colsample_bylevel=1, colsample_bynode=1, \n                reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n                random_state=0, seed=None, missing=None, importance_type='gain')\n# 242 trees appears to be best","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now tune tree specific parameters\n# Tune the minimum child weight and the tree depth\ntrain_sample = train[train.type=='1JHN']\nchild_weight_array = [0, 1, 2, 5, 10]\ndepth_array = [2, 3, 5, 9]\nt_LMAE_inner = []; t_LMAE_outer = []; v_LMAE_inner = []; v_LMAE_outer = [];\nfor cw in child_weight_array:\n    print('For child weight: '+str(cw))\n    for depth in depth_array:\n        print('For depth: '+str(depth))\n        model_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n                max_depth=depth, learning_rate=0.25, n_estimators=242, \n                verbosity=1, \n                objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n                n_jobs=4, \n                gamma=0, min_child_weight=cw, max_delta_step=0, \n                subsample=0.8, colsample_bytree=0.2, colsample_bylevel=1, colsample_bynode=1, \n                reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n                random_state=0, seed=None, missing=None, importance_type='gain')\n        t_LMAE_inner.append(t_LMAE)\n        v_LMAE_inner.append(v_LMAE)\n    t_LMAE_outer.append(t_LMAE_inner)\n    v_LMAE_outer.append(v_LMAE_inner)\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Child weight = 2 and tree depth = 5 give the lowese LMAE: -0.167\nmin(min(v_LMAE_outer))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now tune max features\nfeature_array = [0.1, 0.2, 0.5, 0.8, 1]\nfor feature in feature_array:\n    print('For num features: '+str(feature))\n    model_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n            max_depth=5, learning_rate=0.3, n_estimators=160, \n            verbosity=1, \n            objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n            n_jobs=4, \n            gamma=0, min_child_weight=2, max_delta_step=0, \n            subsample=0.8, colsample_bytree=feature, colsample_bylevel=1, colsample_bynode=1, \n            reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n            random_state=0, seed=None, missing=None, importance_type='gain')\n    t_LMAE_inner.append(t_LMAE)\n    v_LMAE_inner.append(v_LMAE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Best at colsample_bytree = 1 LMAE: -0.355\nmin(v_LMAE_inner)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Now tune subsample\nsubsample_array = [0.1, 0.2, 0.5, 0.8, 1]\nfor subsample in subsample_array:\n    print('For subsample: '+str(subsample))\n    model_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n            max_depth=5, learning_rate=0.3, n_estimators=160, \n            verbosity=1, \n            objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n            n_jobs=4, \n            gamma=0, min_child_weight=2, max_delta_step=0, \n            subsample=subsample, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, \n            reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n            random_state=0, seed=None, missing=None, importance_type='gain')\n    t_LMAE_inner.append(t_LMAE)\n    v_LMAE_inner.append(v_LMAE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Best at subsample = 1 LMAE: -0.3622","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Try one-fifth learning rate and 5x trees\nmodel_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n        max_depth=5, learning_rate=0.06, n_estimators=800, \n        verbosity=1, \n        objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n        n_jobs=4, \n        gamma=0, min_child_weight=2, max_delta_step=0, \n        subsample=1, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, \n        reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n        random_state=0, seed=None, missing=None, importance_type='gain')\nprint('LMAE with 1/5th rate (0.06) and 5x trees (800): '+str(v_LMAE))\n\n# Try one-tenth learning rate and 10x trees\nmodel_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n        max_depth=5, learning_rate=0.03, n_estimators=1600, \n        verbosity=1, \n        objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n        n_jobs=4, \n        gamma=0, min_child_weight=2, max_delta_step=0, \n        subsample=1, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, \n        reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n        random_state=0, seed=None, missing=None, importance_type='gain')\nprint('LMAE with 1/5th rate (0.03) and 5x trees (1600): '+str(v_LMAE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Try old model parameters without k-fold\nmodel_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n        max_depth=11, learning_rate=0.1, n_estimators=3000, \n        verbosity=1, \n        objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n        n_jobs=4, \n        gamma=0, min_child_weight=1, max_delta_step=0, \n        subsample=1, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, \n        reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n        random_state=0, seed=None, missing=None, importance_type='gain')\nprint('LMAE with old parameters without k-fold: '+str(v_LMAE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_1JHN,t_LMAE,v_LMAE=SingleRun(train_sample,fc1,test_size=0.2,model_fn=XGBRegressor,includeType=False,early_stopping_rounds=None,\n        max_depth=11, learning_rate=0.05, n_estimators=6000, \n        verbosity=1, \n        objective='reg:squarederror', booster='gbtree', eval_metric='mae',\n        n_jobs=4, \n        gamma=0, min_child_weight=1, max_delta_step=0, \n        subsample=1, colsample_bytree=1, colsample_bylevel=1, colsample_bynode=1, \n        reg_alpha=0, reg_lambda=1, scale_pos_weight=1, base_score=0.5, \n        random_state=0, seed=None, missing=None, importance_type='gain')\nprint('LMAE with old parameters without k-fold even slowe: '+str(v_LMAE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}