{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport time\nimport gc\nimport pickle\n\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Input, Activation\nfrom keras.layers import BatchNormalization,Add,Dropout\nfrom keras.optimizers import Adam\nfrom keras.models import Model, load_model\nfrom keras import callbacks\nfrom keras import backend as K\nfrom keras.layers.advanced_activations import LeakyReLU\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"dtypes = {'atom_index_0':'uint8', \n          'atom_index_1':'uint8', \n          'scalar_coupling_constant':'float32', \n          'num_C':'uint8', \n          'num_H':'uint8', \n          'num_N':'uint8', \n          'num_O':'uint8', \n          'num_F':'uint8',\n          'total_atoms':'uint8',\n          'num_bonds':'uint8', \n          'num_mol_bonds':'uint8', \n          'min_d':'float32', \n          'mean_d':'float32', \n          'max_d':'float32', \n          'space_dr':'float32', \n          'bond_dr':'float32',\n          'bond_1':'uint8', \n          'bond_2':'uint8', \n          'bond_3':'uint8', \n          'atom_0_pc':'float32', \n          'atom_end_pc':'float32',\n          'atom_2_hyb':'uint8', \n          'atom_3_hyb':'uint8', \n          'atom_end_hyb':'uint8', \n          'path_count':'uint8', \n          'atom_0_min':'float32',\n          'atom_0_mean':'float32', \n          'atom_0_max':'float32', \n          'atom_0_Cmin':'float32', \n          'atom_0_Cmean':'float32',\n          'atom_0_Cmax':'float32', \n          'atom_0_Omin':'float32', \n          'atom_0_Omean':'float32',\n          'atom_0_Omax':'float32', \n          'atom_0_Nmin':'float32', \n          'atom_0_Nmean':'float32', \n          'atom_0_Nmax':'float32',\n          'atom_0_Fmin':'float32', \n          'atom_0_Fmean':'float32', \n          'atom_0_Fmax':'float32', \n          'atom_end_min':'float32',\n          'atom_end_mean':'float32', \n          'atom_end_max':'float32', \n          'atom_end_Cmin':'float32', \n          'atom_end_Cmean':'float32',\n          'atom_end_Cmax':'float32', \n          'atom_end_Omin':'float32', \n          'atom_end_Omean':'float32',\n          'atom_end_Omax':'float32', \n          'atom_end_Nmin':'float32', \n          'atom_end_Nmean':'float32', \n          'atom_end_Nmax':'float32',\n          'atom_end_Fmin':'float32', \n          'atom_end_Fmean':'float32', \n          'atom_end_Fmax':'float32',\n          'Dmin_COM':'float32', \n          'Dmean_COM':'float32', \n          'Dmax_COM':'float32',\n          'COM_dr_0': 'float32',\n          'COM_dr_1': 'float32',\n          'bond2_angle': 'float32',\n          'bond3_angle': 'float32'\n         }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/predmolprop-featureengineering-finaltrain/train_extend.csv\",dtype=dtypes)\ntest = pd.read_csv(\"../input/predmolprop-featureengineering-finaltest/test_extend.csv\",dtype=dtypes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PopList = ['molecule_name', 'atom_index_0','atom_index_1','num_bonds','atom_end_type',\n           'bond_1','is_linear','atom_2_hyb','atom_3_hyb','atom_end_hyb',\n           'atom_0_fc', 'atom_end_fc','atom_0_val','atom_end_val','atom_0_sm', 'atom_end_sm']\n\nfor col in PopList:\n    if col in train:\n        train.pop(col)\n        test.pop(col)\n    \ntrain.fillna(value ='',inplace= True)\ntest.fillna(value='',inplace=True)\n\ncoupling_types = sorted(list(train.type.unique()))\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encode categorical features\nfrom sklearn.preprocessing import LabelEncoder\n\ncols = ['atom_0_type2','atom_2_type','atom_3_type','atom_end_type2']\nfor col in cols:\n    enc = LabelEncoder()\n    train[col]=enc.fit_transform(train[col]).astype(np.uint8)\n    test[col]=enc.transform(test[col]).astype(np.uint8)\ndel cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature lists\nprefix_train= ['id', 'type', 'scalar_coupling_constant']\nprefix_test= ['id', 'type']\n\nfc_distance = ['space_dr','bond_dr','min_d','mean_d', 'max_d']\n\nfc_COM = ['Dmin_COM', 'Dmean_COM', 'Dmax_COM']\n\nfc_size = ['num_mol_bonds', 'total_atoms','num_C', 'num_H', 'num_N', 'num_O', 'num_F']\n\nfc_atom_0 = ['atom_0_pc','atom_0_type2','COM_dr_0',\n             'atom_0_min','atom_0_mean', 'atom_0_max',\n             'atom_0_Cmin', 'atom_0_Cmean','atom_0_Cmax',\n             'atom_0_Nmin', 'atom_0_Nmean', 'atom_0_Nmax',\n             'atom_0_Omin', 'atom_0_Omean','atom_0_Omax',\n             'atom_0_Fmin', 'atom_0_Fmean', 'atom_0_Fmax']\n\nfc_atom_end = ['atom_end_pc', 'atom_end_type2', 'COM_dr_1',\n               'atom_end_min','atom_end_mean', 'atom_end_max',\n               'atom_end_Cmin', 'atom_end_Cmean','atom_end_Cmax',\n               'atom_end_Nmin', 'atom_end_Nmean', 'atom_end_Nmax',\n               'atom_end_Omin', 'atom_end_Omean','atom_end_Omax',\n               'atom_end_Fmin', 'atom_end_Fmean', 'atom_end_Fmax']\n\nfc_2 = ['path_count','atom_2_type','bond_2','bond2_angle']\nfc_3 = ['atom_3_type','bond_3','bond3_angle']\n\nfc1 = set(fc_distance+fc_COM+fc_size+fc_atom_0+fc_atom_end)\n\n# reorder\ntrain = train[prefix_train+fc_distance+fc_COM+fc_size+fc_atom_0+fc_atom_end+fc_2+fc_3]\ntest = test[prefix_test+fc_distance+fc_COM+fc_size+fc_atom_0+fc_atom_end+fc_2+fc_3]\nlen(train.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process Data\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import StandardScaler\ndef ProcessData(df,features,test_size=0.25):\n    scaler = StandardScaler()\n    features = sorted(list(features))\n    if test_size == 0:\n        train_Y = df.pop('scalar_coupling_constant')\n        train_type = df.pop('type')\n        df.pop('id')\n        train_X[features] = scaler.fit_transform(df[features])\n        return train_X, train_Y, train_type, scaler\n        #return df.loc[:,df.columns.map(lambda x: x in features)], train_Y, train_type\n    \n    train_X, val_X, train_Y, val_Y = train_test_split(df[features+['type']], \n                                                      df.scalar_coupling_constant,test_size=test_size,random_state=42)\n    \n    train_type = train_X.pop('type')\n    val_type = val_X.pop('type')\n    scaler.fit(train_X)#.append(val_X))\n    train_X[train_X.columns] = scaler.transform(train_X[train_X.columns])\n    val_X[val_X.columns] = scaler.transform(val_X[val_X.columns])\n    return train_X, train_Y, train_type, val_X, val_Y, val_type, scaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def CalcLMAE(y_true, y_pred, groups, floor=1e-9):\n    maes = (y_true-y_pred).abs().groupby(groups).mean()\n    return np.log(maes.map(lambda x: max(x, floor))).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def BuildKerasModel(input_shape,hidden_units=[1024,128,4],dropout_rate=0.2,lr=0.001,beta1=0.9,beta2=0.999):\n    model=Sequential()\n    model.add(Dense(hidden_units[0], activation = 'relu', input_shape=input_shape))\n    #model.add(BatchNormalization())\n    if(dropout_rate>0):\n        model.add(Dropout(rate=dropout_rate))\n    for units in hidden_units[1:]:\n        model.add(Dense(units, activation = 'relu'))\n        #model.add(BatchNormalization())\n        if(dropout_rate>0):\n            model.add(Dropout(rate=dropout_rate))\n    model.add(Dense(1))\n    model.compile(loss='mean_absolute_error', \n                  optimizer=Adam(learning_rate=lr, beta_1=beta1, beta_2=beta2))\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_history(history, label):\n    for key in sorted(history.history.keys()):\n        plt.plot(history.history[key])\n        \n    plt.title('Loss for %s' % label)\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    _= plt.legend(list(sorted(history.history.keys())), loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def SingleRun(df,features,hidden_units=[1024,512,128],dropout=0.2,lr=0.001,beta1=0.9,beta2=0.999,\n              test_size=0.25,includeType=False,DoCallbacks=False,**kwargs):\n    data = ProcessData(df,features,test_size)\n    if(test_size==0):\n        train_X,train_Y,train_type,scaler = data\n    else:\n        train_X,train_Y,train_type,val_X,val_Y,val_type,scaler = data\n    if includeType:\n        train_X=train_X.join(train_type)\n        val_X=val_X.join(val_type)\n        enc = LabelEncoder()\n        train_X.type=enc.fit_transform(train_X.type).astype(np.uint8)\n        val_X.type=enc.transform(val_X.type).astype(np.uint8)\n    \n    model=BuildKerasModel((len(features),),hidden_units=hidden_units,dropout_rate=dropout,lr=lr,beta1=beta1,beta2=beta2)\n    \n    t1=time.time()\n    if DoCallbacks:\n        if test_size==0:\n            print('ERROR: need test data for early_stopping_rounds')\n            return -1,-1,-1\n    \n        # Callback for Early Stopping... May want to raise the min_delta for small numbers of epochs\n        es = callbacks.EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=20,verbose=1, \n                                     mode='min', restore_best_weights=True)\n        # Callback for Reducing the Learning Rate... when the monitor levels out for 'patience' epochs, then the LR is reduced\n        rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.1,patience=10, \n                                          min_lr=1e-6, mode='min', verbose=1)\n\n        history=model.fit(x=train_X, y=train_Y, callbacks=[rlr,es], validation_data=(val_X,val_Y), **kwargs)\n    else:\n        if test_size==0:\n            history=model.fit(x=train_X, y=train_Y, **kwargs)\n        else:\n            history=model.fit(x=train_X, y=train_Y, validation_data=(val_X,val_Y), **kwargs)\n    \n    plot_history(history, train_type.values[0])\n        \n    t2=time.time()\n    print('training time:',t2-t1)\n    train_predict = pd.Series(model.predict(train_X).flatten(),\n                              index=train_X.index)\n    train_LMAE = CalcLMAE(train_Y,train_predict,train_type)\n    print('\\ttrain LMAE:',train_LMAE)\n    if(test_size==0):\n        val_LMAE = None\n        plt.figure(figsize=[6,6])\n        sns.scatterplot(x=train_Y, y=train_predict)\n        plt.plot(val_Y,val_Y,color='black')\n        plt.title(train_type.values[0])\n        plt.xlabel('scalar coupling constant')\n        plt.ylabel('predicted value')\n#        lims = plt.xlim()\n#        width=lims[1]-lims[0]\n#        lims=lims[0]-0.01*width,lims[1]+0.01*width\n#        plt.xlim(lims)\n#        plt.ylim(lims)\n        plt.show()\n    else:\n        val_predict = pd.Series(model.predict(val_X).flatten(),index=val_X.index)\n        val_LMAE = CalcLMAE(val_Y,val_predict,val_type)\n        print('\\tval LMAE:',val_LMAE)\n        plt.figure(figsize=[6,6])\n        sns.scatterplot(x=val_Y, y=val_predict)\n        plt.plot(val_Y,val_Y,color='black')\n        plt.title(train_type.values[0])\n        plt.xlabel('scalar coupling constant')\n        plt.ylabel('predicted value')\n#        lims = plt.xlim()\n#        width=lims[1]-lims[0]\n#        lims=lims[0]-0.01*width,lims[1]+0.01*width\n#        plt.xlim(lims)\n#        plt.ylim(lims)\n        plt.show()\n        \n    gc.collect()\n    return model, train_LMAE, val_LMAE, scaler","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"Jtype = '2JHN'\ntrain_sample = train[train.type==Jtype].sample(10000)\nfeatures=fc1.copy()\nfeatures.update(fc_2)\n#features.update(fc_2[:-1]+fc_3)\n\nmodel, train_LMAE, val_LMAE, scaler = SingleRun(train_sample,features,\n                                                hidden_units=[128,512,128,64,32],\n                                                dropout=0,lr=0.001,beta1=0.9,beta2=0.999,\n                                                test_size=0.2,includeType=False,DoCallbacks=True,# end of params, start kwargs,\n                                                batch_size=128, epochs=200, verbose=0,shuffle=True,\n                                                validation_steps=None, validation_freq=1)\n\n#model.save('model_'+Jtype)\n#pickle.dump(scaler,'scaler_'+Jtype)"},{"metadata":{},"cell_type":"markdown","source":"**RUN BY TYPE**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def RunByType(df,hidden_units=[1024,512,128],dropout=0,lr=0.001,beta1=0.9,beta2=0.999,\n              test_size=0.25,includeType=False,DoCallbacks=False,**kwargs):\n    model_dict={}\n    train_LMAE_dict={}\n    val_LMAE_dict={}\n    scaler_dict={}\n    for coupling_type in coupling_types:\n        print('Now training type:',str(coupling_type))\n\n        df_type = df[df['type']==coupling_type]\n\n        features=fc1.copy()\n        if(coupling_type[0]=='2'):\n            features.update(fc_2)\n        elif(coupling_type[0]=='3'):\n            features.update(fc_2[:-1]+fc_3)\n        \n        model,train_LMAE,val_LMAE,scaler = SingleRun(df_type,features,\n                                              hidden_units=hidden_units,dropout=dropout,lr=lr,beta1=beta1,beta2=beta2,\n                                              test_size=test_size,includeType=includeType,\n                                              DoCallbacks=DoCallbacks,**kwargs)\n\n        model_dict[coupling_type]=model\n        train_LMAE_dict[coupling_type]=train_LMAE\n        val_LMAE_dict[coupling_type]=val_LMAE\n        scaler_dict[coupling_type]=scaler\n        \n    train_LMAE = sum(train_LMAE_dict.values())/len(coupling_types)\n    if test_size>0:\n        val_LMAE = sum(val_LMAE_dict.values())/len(coupling_types)\n    else:\n        val_LMAE=None\n    print('total train LMAE:',train_LMAE)\n    print('total val LMAE', val_LMAE)\n    gc.collect()\n    return model_dict, scaler_dict#,train_LMAE, val_LMAE\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def PredictByType(df_X,model_dict,scaler_dict,df_Y=None):\n    predictions = pd.DataFrame()\n\n    for coupling_type in coupling_types:\n        print('predicting type:',str(coupling_type))\n        model = model_dict[coupling_type]\n        scaler = scaler_dict[coupling_type]\n        df_type = df_X[df_X['type']==coupling_type]\n        \n        features=fc1.copy()\n        if(coupling_type[0]=='2'):\n            features.update(fc_2)\n        elif(coupling_type[0]=='3'):\n            features.update(fc_2[:-1]+fc_3)\n        \n        features = sorted(list(features))\n        predict = model.predict(scaler.transform(df_type[features])).flatten()\n        predict = pd.Series(predict,index=df_type.index)\n        predict = pd.DataFrame({'id':df_type.id,'scalar_coupling_constant': predict})\n        predict.set_index('id',inplace=True)\n        predictions = pd.concat([predictions,predict])\n    \n    if df_Y is not None:\n        predictions.columns = ['predictions']\n        merged = pd.merge(train,predictions,how='left',left_on='id',right_index=True)\n        LMAE = CalcLMAE(df_Y, merged.predictions,merged.type)\n        print('LMAE:', LMAE)\n        g = sns.FacetGrid(merged,col=\"type\", col_order = coupling_types,sharex=False,sharey=False)\n        g.map(sns.scatterplot, \"scalar_coupling_constant\",\"predictions\")\n        del merged\n    \n    gc.collect()\n    return predictions\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_dict,scaler_dict=RunByType(train,\n                     hidden_units=[128,512,512,128,64,32],dropout=0,lr=0.001,beta1=0.9,beta2=0.999,\n                     test_size=0.2,\n                     includeType=False,\n                     DoCallbacks=True, # end of params, start kwargs\n                     batch_size=128, epochs=200, verbose=0,\n                     shuffle=True,\n                     validation_steps=None, validation_freq=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predict = PredictByType(test,model_dict,scaler_dict)\ntest_predict.to_csv('test_predict_TypeModel.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}