{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Predicting Molecular properties : Keras DNN\n\nHello everyone ! This is my first public Kernel in Kaggle.\nI have passed the last months on Kaggle trying to improve my data science skills and I still have a lot to learn.\nSo why share my work and I am just a new explorer of this deep field ? To get your advice about how to improve my work and maybe help people making their first steps here.\n\nEnough talking, in this Kernel I will mainly use a DNN after making some feature engineering.\nLet's start with imports.\n\n### 1. Load the main data sets"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/champs-scalar-coupling/train.csv')\ntest = pd.read_csv('../input/champs-scalar-coupling/test.csv')\nsub = pd.read_csv('../input/champs-scalar-coupling/sample_submission.csv')\nstructures = pd.read_csv('../input/champs-scalar-coupling/structures.csv')\ntrain_sub_charge=pd.read_csv('../input/champs-scalar-coupling/mulliken_charges.csv')\ntrain_sub_tensor=pd.read_csv('../input/champs-scalar-coupling/magnetic_shielding_tensors.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This call allows to **reduce memory usage**, which is really necessary for this Kernel to be run on small cloud instances.\n\nWhat this call basicly does is to fit the values of each column to the numerical type that uses the least memory possible.\n\nRef: [https://www.kaggle.com/todnewman/keras-neural-net-for-champs](https://www.kaggle.com/todnewman/keras-neural-net-for-champs)"},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\nprint(train.shape, test.shape, structures.shape)\ntrain = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)\nstructures = reduce_mem_usage(structures)\ntrain_sub_charge = reduce_mem_usage(train_sub_charge)\ntrain_sub_tensor = reduce_mem_usage(train_sub_tensor)\nprint(train.shape, test.shape, structures.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'There are {train.shape[0]} rows in train data.')\nprint(f'There are {test.shape[0]} rows in test data.')\n\nprint(f\"There are {train['molecule_name'].nunique()} distinct molecules in train data.\")\nprint(f\"There are {test['molecule_name'].nunique()} distinct molecules in test data.\")\nprint(f\"There are {structures['atom'].nunique()} unique atoms.\")\nprint(f\"There are {train['type'].nunique()} unique types.\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2. Feature engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_=train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#add atom coords\ndef map_atom_info(df, atom_idx):\n    df = pd.merge(df, structures, how = 'left',\n                  left_on  = ['molecule_name', f'atom_index_{atom_idx}'],\n                  right_on = ['molecule_name',  'atom_index'])\n    \n    df = df.drop('atom_index', axis=1)\n    df = df.rename(columns={'atom': f'atom_{atom_idx}',\n                            'x': f'x_{atom_idx}',\n                            'y': f'y_{atom_idx}',\n                            'z': f'z_{atom_idx}'})\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"''' \nMap atom info from the structures.csv into the train/test files\n'''\nimport psutil\nimport os\n\ndef map_atom_info(df_1,df_2, atom_idx):\n    print('Mapping...', df_1.shape, df_2.shape, atom_idx)\n    \n    df = pd.merge(df_1, df_2.drop_duplicates(subset=['molecule_name', 'atom_index']), how = 'left',\n                  left_on  = ['molecule_name', f'atom_index_{atom_idx}'],\n                  right_on = ['molecule_name',  'atom_index'])\n    \n    df = df.drop('atom_index', axis=1)\n\n    return df\n\ndef show_ram_usage():\n    py = psutil.Process(os.getpid())\n    print('RAM usage: {} GB'.format(py.memory_info()[0]/2. ** 30))\n\nshow_ram_usage()\n\nfor atom_idx in [0,1]:\n    train = map_atom_info(train,structures, atom_idx)\n    train = map_atom_info(train,train_sub_charge, atom_idx)\n    train = map_atom_info(train,train_sub_tensor, atom_idx)\n    train = train.rename(columns={'atom': f'atom_{atom_idx}',\n                                        'x': f'x_{atom_idx}',\n                                        'y': f'y_{atom_idx}',\n                                        'z': f'z_{atom_idx}',\n                                        'mulliken_charge': f'charge_{atom_idx}',\n                                        'XX': f'XX_{atom_idx}',\n                                        'YX': f'YX_{atom_idx}',\n                                        'ZX': f'ZX_{atom_idx}',\n                                        'XY': f'XY_{atom_idx}',\n                                        'YY': f'YY_{atom_idx}',\n                                        'ZY': f'ZY_{atom_idx}',\n                                        'XZ': f'XZ_{atom_idx}',\n                                        'YZ': f'YZ_{atom_idx}',\n                                        'ZZ': f'ZZ_{atom_idx}',})\n\n    test = map_atom_info(test,structures, atom_idx)\n    test = test.rename(columns={'atom': f'atom_{atom_idx}',\n                                'x': f'x_{atom_idx}',\n                                'y': f'y_{atom_idx}',\n                                'z': f'z_{atom_idx}'})\n    #add some features\n    \n    structures['c_x']=structures.groupby('molecule_name')['x'].transform('mean')\n    structures['c_y']=structures.groupby('molecule_name')['y'].transform('mean')\n    structures['c_z']=structures.groupby('molecule_name')['z'].transform('mean')\n    structures['atom_n']=structures.groupby('molecule_name')['atom_index'].transform('max')\n    \n    show_ram_usage()\n    print(train.shape, test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"#create distance feature\ndef make_features(df):\n    df['dx']=df['x_1']-df['x_0']\n    df['dy']=df['y_1']-df['y_0']\n    df['dz']=df['z_1']-df['z_0']\n    df['distance']=(df['dx']**2+df['dy']**2+df['dz']**2)**(1/2)\n    return df\n\ntrain_=make_features(train)\ntest=make_features(test)\ntest_prediction=np.zeros(len(test))\nshow_ram_usage()\nprint(train_.shape, test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#create more complex feature\ndef get_dist(df):\n    df_temp=df.loc[:,[\"molecule_name\",\"atom_index_0\",\"atom_index_1\",\"distance\",\"x_0\",\"y_0\",\"z_0\",\"x_1\",\"y_1\",\"z_1\"]].copy()\n    df_temp_=df_temp.copy()\n    df_temp_= df_temp_.rename(columns={'atom_index_0': 'atom_index_1',\n                                       'atom_index_1': 'atom_index_0',\n                                       'x_0': 'x_1',\n                                       'y_0': 'y_1',\n                                       'z_0': 'z_1',\n                                       'x_1': 'x_0',\n                                       'y_1': 'y_0',\n                                       'z_1': 'z_0'})\n    df_temp_all=pd.concat((df_temp,df_temp_),axis=0)\n\n    df_temp_all[\"min_distance\"]=df_temp_all.groupby(['molecule_name', 'atom_index_0'])['distance'].transform('min')\n    df_temp_all[\"max_distance\"]=df_temp_all.groupby(['molecule_name', 'atom_index_0'])['distance'].transform('max')\n    \n    df_temp= df_temp_all[df_temp_all[\"min_distance\"]==df_temp_all[\"distance\"]].copy()\n    df_temp=df_temp.drop(['x_0','y_0','z_0','min_distance'], axis=1)\n    df_temp= df_temp.rename(columns={'atom_index_0': 'atom_index',\n                                         'atom_index_1': 'atom_index_closest',\n                                         'distance': 'distance_closest',\n                                         'x_1': 'x_closest',\n                                         'y_1': 'y_closest',\n                                         'z_1': 'z_closest'})\n    \n    for atom_idx in [0,1]:\n        df = map_atom_info(df,df_temp, atom_idx)\n        df = df.rename(columns={'atom_index_closest': f'atom_index_closest_{atom_idx}',\n                                        'distance_closest': f'distance_closest_{atom_idx}',\n                                        'x_closest': f'x_closest_{atom_idx}',\n                                        'y_closest': f'y_closest_{atom_idx}',\n                                        'z_closest': f'z_closest_{atom_idx}'})\n        \n    df_temp= df_temp_all[df_temp_all[\"max_distance\"]==df_temp_all[\"distance\"]].copy()\n    df_temp=df_temp.drop(['x_0','y_0','z_0','max_distance'], axis=1)\n    df_temp= df_temp.rename(columns={'atom_index_0': 'atom_index',\n                                         'atom_index_1': 'atom_index_farthest',\n                                         'distance': 'distance_farthest',\n                                         'x_1': 'x_farthest',\n                                         'y_1': 'y_farthest',\n                                         'z_1': 'z_farthest'})\n        \n    for atom_idx in [0,1]:\n        df = map_atom_info(df,df_temp, atom_idx)\n        df = df.rename(columns={'atom_index_farthest': f'atom_index_farthest_{atom_idx}',\n                                        'distance_farthest': f'distance_farthest_{atom_idx}',\n                                        'x_farthest': f'x_farthest_{atom_idx}',\n                                        'y_farthest': f'y_farthest_{atom_idx}',\n                                        'z_farthest': f'z_farthest_{atom_idx}'})\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create more complex feature\ntest=(get_dist(test)) \ntrain=(get_dist(train))\n\nprint(train.shape, test.shape)\nshow_ram_usage()","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"#create cosinus distance\ndef add_features(df):\n    df[\"distance_center0\"]=((df['x_0']-df['c_x'])**2+(df['y_0']-df['c_y'])**2+(df['z_0']-df['c_z'])**2)**(1/2)\n    df[\"distance_center1\"]=((df['x_1']-df['c_x'])**2+(df['y_1']-df['c_y'])**2+(df['z_1']-df['c_z'])**2)**(1/2)\n    df[\"distance_c0\"]=((df['x_0']-df['x_closest_0'])**2+(df['y_0']-df['y_closest_0'])**2+(df['z_0']-df['z_closest_0'])**2)**(1/2)\n    df[\"distance_c1\"]=((df['x_1']-df['x_closest_1'])**2+(df['y_1']-df['y_closest_1'])**2+(df['z_1']-df['z_closest_1'])**2)**(1/2)\n    df[\"distance_f0\"]=((df['x_0']-df['x_farthest_0'])**2+(df['y_0']-df['y_farthest_0'])**2+(df['z_0']-df['z_farthest_0'])**2)**(1/2)\n    df[\"distance_f1\"]=((df['x_1']-df['x_farthest_1'])**2+(df['y_1']-df['y_farthest_1'])**2+(df['z_1']-df['z_farthest_1'])**2)**(1/2)\n    df[\"vec_center0_x\"]=(df['x_0']-df['c_x'])/(df[\"distance_center0\"]+1e-10)\n    df[\"vec_center0_y\"]=(df['y_0']-df['c_y'])/(df[\"distance_center0\"]+1e-10)\n    df[\"vec_center0_z\"]=(df['z_0']-df['c_z'])/(df[\"distance_center0\"]+1e-10)\n    df[\"vec_center1_x\"]=(df['x_1']-df['c_x'])/(df[\"distance_center1\"]+1e-10)\n    df[\"vec_center1_y\"]=(df['y_1']-df['c_y'])/(df[\"distance_center1\"]+1e-10)\n    df[\"vec_center1_z\"]=(df['z_1']-df['c_z'])/(df[\"distance_center1\"]+1e-10)\n    df[\"vec_c0_x\"]=(df['x_0']-df['x_closest_0'])/(df[\"distance_c0\"]+1e-10)\n    df[\"vec_c0_y\"]=(df['y_0']-df['y_closest_0'])/(df[\"distance_c0\"]+1e-10)\n    df[\"vec_c0_z\"]=(df['z_0']-df['z_closest_0'])/(df[\"distance_c0\"]+1e-10)\n    df[\"vec_c1_x\"]=(df['x_1']-df['x_closest_1'])/(df[\"distance_c1\"]+1e-10)\n    df[\"vec_c1_y\"]=(df['y_1']-df['y_closest_1'])/(df[\"distance_c1\"]+1e-10)\n    df[\"vec_c1_z\"]=(df['z_1']-df['z_closest_1'])/(df[\"distance_c1\"]+1e-10)\n    df[\"vec_f0_x\"]=(df['x_0']-df['x_farthest_0'])/(df[\"distance_f0\"]+1e-10)\n    df[\"vec_f0_y\"]=(df['y_0']-df['y_farthest_0'])/(df[\"distance_f0\"]+1e-10)\n    df[\"vec_f0_z\"]=(df['z_0']-df['z_farthest_0'])/(df[\"distance_f0\"]+1e-10)\n    df[\"vec_f1_x\"]=(df['x_1']-df['x_farthest_1'])/(df[\"distance_f1\"]+1e-10)\n    df[\"vec_f1_y\"]=(df['y_1']-df['y_farthest_1'])/(df[\"distance_f1\"]+1e-10)\n    df[\"vec_f1_z\"]=(df['z_1']-df['z_farthest_1'])/(df[\"distance_f1\"]+1e-10)\n    df[\"vec_x\"]=(df['x_1']-df['x_0'])/df[\"distance\"]\n    df[\"vec_y\"]=(df['y_1']-df['y_0'])/df[\"distance\"]\n    df[\"vec_z\"]=(df['z_1']-df['z_0'])/df[\"distance\"]\n    df[\"cos_c0_c1\"]=df[\"vec_c0_x\"]*df[\"vec_c1_x\"]+df[\"vec_c0_y\"]*df[\"vec_c1_y\"]+df[\"vec_c0_z\"]*df[\"vec_c1_z\"]\n    df[\"cos_f0_f1\"]=df[\"vec_f0_x\"]*df[\"vec_f1_x\"]+df[\"vec_f0_y\"]*df[\"vec_f1_y\"]+df[\"vec_f0_z\"]*df[\"vec_f1_z\"]\n    df[\"cos_center0_center1\"]=df[\"vec_center0_x\"]*df[\"vec_center1_x\"]+df[\"vec_center0_y\"]*df[\"vec_center1_y\"]+df[\"vec_center0_z\"]*df[\"vec_center1_z\"]\n    df[\"cos_c0\"]=df[\"vec_c0_x\"]*df[\"vec_x\"]+df[\"vec_c0_y\"]*df[\"vec_y\"]+df[\"vec_c0_z\"]*df[\"vec_z\"]\n    df[\"cos_c1\"]=df[\"vec_c1_x\"]*df[\"vec_x\"]+df[\"vec_c1_y\"]*df[\"vec_y\"]+df[\"vec_c1_z\"]*df[\"vec_z\"]\n    df[\"cos_f0\"]=df[\"vec_f0_x\"]*df[\"vec_x\"]+df[\"vec_f0_y\"]*df[\"vec_y\"]+df[\"vec_f0_z\"]*df[\"vec_z\"]\n    df[\"cos_f1\"]=df[\"vec_f1_x\"]*df[\"vec_x\"]+df[\"vec_f1_y\"]*df[\"vec_y\"]+df[\"vec_f1_z\"]*df[\"vec_z\"]\n    df[\"cos_center0\"]=df[\"vec_center0_x\"]*df[\"vec_x\"]+df[\"vec_center0_y\"]*df[\"vec_y\"]+df[\"vec_center0_z\"]*df[\"vec_z\"]\n    df[\"cos_center1\"]=df[\"vec_center1_x\"]*df[\"vec_x\"]+df[\"vec_center1_y\"]*df[\"vec_y\"]+df[\"vec_center1_z\"]*df[\"vec_z\"]\n    df=df.drop(['vec_c0_x','vec_c0_y','vec_c0_z','vec_c1_x','vec_c1_y','vec_c1_z',\n                'vec_f0_x','vec_f0_y','vec_f0_z','vec_f1_x','vec_f1_y','vec_f1_z',\n                'vec_center0_x','vec_center0_y','vec_center0_z','vec_center1_x','vec_center1_y','vec_center1_z',\n                'vec_x','vec_y','vec_z'], axis=1)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create cosinus distance\ntrain=add_features(train)\ntest=add_features(test)\nprint(train.shape, test.shape)\nshow_ram_usage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#quantitative distribution\nquantitative = [f for f in train_.columns if train_.dtypes[f] != 'object']\nquantitative.remove('scalar_coupling_constant')\nquantitative.remove('id')\n#qualitative ditribution\nqualitative = [f for f in train_.columns if train_.dtypes[f] == 'object']\nqualitative.remove('molecule_name')\nfor c in qualitative:\n    train_[c] = train_[c].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print qualitative features\nqualitative","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['type_0'] = train['type'].apply(lambda x: x[0])\ntest['type_0'] = test['type'].apply(lambda x: x[0])\ntrain['type_1'] = train['type'].apply(lambda x: x[1:])\ntest['type_1'] = test['type'].apply(lambda x: x[1:])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.columns.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3. Investigate features created"},{"metadata":{"trusted":true},"cell_type":"code","source":"#scatter plot cos_c0/scalar_coupling_constant\nvar = 'cos_c0'\nfor ctype in train['type'].unique() :\n    train_plot  = train.loc[train['type']==ctype][:3000]\n    data = pd.concat([train_plot['scalar_coupling_constant'], train_plot[var]], axis=1)\n    data.plot.scatter(x=var, y='scalar_coupling_constant');\n    plt.title(f'{ctype}', fontsize=18)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#histogram and normal probability plot\nfrom scipy import stats\nfrom scipy.stats import norm\nvar = 'cos_f1'\ntrain_plot  = train[:10000]\nfor ctype in train_.type.unique():\n    plt.figure()\n    plt.title(f'{ctype}', fontsize=18)\n    sns.distplot((train_plot.loc[train['type']==ctype])[var], fit=norm);\n    plt.figure()\n    res = stats.probplot((train_plot.loc[train['type']==ctype])[var], plot=plt)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#correlation matrix with absolute value\ncorrmat = train[:10000].corr()\ncorrmat = corrmat.abs()\nf, ax = plt.subplots(figsize=(12, 9))\nsns.heatmap(corrmat, vmax=.8, square=True);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#scalar_coupling_constant correlation matrix\nk = 10 #number of variables for heatmap\ncols = corrmat.nlargest(k, 'scalar_coupling_constant')['scalar_coupling_constant'].index\ncm = np.corrcoef(train[:10000][cols].values.T)\nsns.set(font_scale=1.25)\nhm = sns.heatmap(cm, cbar=True, annot=True, square=True, fmt='.2f', annot_kws={'size': 10}, yticklabels=cols.values, xticklabels=cols.values)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#LabelEncoder : string labels to integers\nfrom sklearn.preprocessing import LabelEncoder\nfor f in ['atom_0', 'atom_1', 'type_0', 'type_1', 'type']:\n    lbl = LabelEncoder()\n    lbl.fit(list(train[f].values) )\n    train[f] = lbl.transform(list(train[f].values))\n    test[f] = lbl.transform(list(test[f].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#scatter plot totalbsmtsf/saleprice\nvar = 'distance'\nfor ctype in range(0,7):\n    train_plot  = train_.loc[train['type']==ctype][:3000]\n    data = pd.concat([train_plot['scalar_coupling_constant'], train_plot[var]], axis=1)\n    data.plot.scatter(x=var, y='scalar_coupling_constant');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. Training Neural Network"},{"metadata":{},"cell_type":"markdown","source":"#### Extracting a small testing set from the training data :"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ = train[4200000:]\ntrain_ = train[:4200000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X = train_.drop(['id', 'molecule_name', 'scalar_coupling_constant', 'atom_index_0', 'atom_index_1'], axis=1)\ny = train_['scalar_coupling_constant']\nX_test = test.drop(['id', 'molecule_name', 'atom_index_0', 'atom_index_1'], axis=1)\nX_test_ = test_.drop(['id', 'molecule_name','scalar_coupling_constant', 'atom_index_0', 'atom_index_1'], axis=1) #subset of training to test model\ny_verif = test_['scalar_coupling_constant'] #verifying data to test model accuracy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_test.shape , X_test_.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(X_test_.columns.values)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Defining DNN model with Keras:"},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"from keras import callbacks\n# Set to True if we want to train from scratch.  False will reuse saved models as a starting point.\nretrain = True\nmodel_name_rd = ('../keras-neural-net-for-champs/molecule_model.hdf5')\nmodel_name_wrt = ('/kaggle/working/molecule_model.hdf5')\n\nes = callbacks.EarlyStopping(monitor='loss', min_delta=0.1, patience=8,verbose=1, mode='auto', restore_best_weights=True)\n# Callback for Reducing the Learning Rate... when the monitor levels out for 'patience' epochs, then the LR is reduced\nrlr = callbacks.ReduceLROnPlateau(monitor='loss', factor=0.1,patience=7, min_lr=1e-6, min_delta=0.1, mode='auto', verbose=1)\n# Save the best value of the model for future use\nsv_mod = callbacks.ModelCheckpoint(model_name_wrt, monitor='loss', save_best_only=True, period=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.wrappers.scikit_learn import KerasRegressor\nfrom keras.models import Sequential\nfrom sklearn.metrics import accuracy_score\nfrom keras.layers import Dense, Input, Activation\nfrom keras.layers import BatchNormalization,Add,Dropout\nfrom keras.optimizers import Adam\nfrom keras.models import Model, load_model\nfrom keras import callbacks\nfrom keras import backend as K\nfrom keras.layers.advanced_activations import LeakyReLU\nimport eli5\nfrom eli5.sklearn import PermutationImportance\n\ndef baseline_model():\n    model = Sequential()\n    model.add(Dense(256, input_dim=31, activation='relu',kernel_initializer='normal'))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=0.05))\n    model.add(Dropout(0.4))\n    model.add(Dense(1024))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=0.05))\n    model.add(Dropout(0.2))\n    model.add(Dense(256))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=0.05))\n    model.add(Dropout(0.2))\n    model.add(Dense(64))\n    model.add(BatchNormalization())\n    model.add(LeakyReLU(alpha=0.05))\n    model.add(Dropout(0.2))\n    model.add(Dense(1))\n    model.compile(loss='mean_squared_error', optimizer='adam')\n    if not retrain:\n        my_model = load_model(model_name_wrt)\n        return my_model\n    return model\n\nimport time\nstart_time = time.time()\n\nmy_model = KerasRegressor(build_fn=baseline_model, batch_size=3000, epochs=400, verbose=True,  callbacks=[es, rlr, sv_mod])\nmy_model.fit(np.array(X[['type','distance','distance_center0','distance_center1','distance_c0','distance_c1','distance_f0','distance_f1', 'type_0', 'type_1',\n                         'cos_f0', 'cos_f1',  'cos_c0_c1', 'cos_f0_f1','cos_c0', 'cos_c1', \"x_0\",\"y_0\",\"z_0\",\"x_1\",\"y_1\",\"z_1\",\"c_x\",\"c_y\",\"c_z\",\n                    'x_closest_0','y_closest_0','z_closest_0','x_closest_1','y_closest_1','z_closest_1']]),y)\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\n#perm = PermutationImportance(my_model, random_state=1).fit(np.array(X),y)\n#eli5.show_weights(perm, feature_names = X.columns.tolist())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Predicting on the extracted test set :"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_y = my_model.predict(np.array(X_test_[['type','distance','distance_center0','distance_center1','distance_c0','distance_c1','distance_f0','distance_f1', 'type_0', 'type_1',\n                         'cos_f0', 'cos_f1',  'cos_c0_c1', 'cos_f0_f1','cos_c0', 'cos_c1', \"x_0\",\"y_0\",\"z_0\",\"x_1\",\"y_1\",\"z_1\",\"c_x\",\"c_y\",\"c_z\",\n                    'x_closest_0','y_closest_0','z_closest_0','x_closest_1','y_closest_1','z_closest_1']]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#metric for this competition\nfrom sklearn import metrics\ndef metric(df, preds):\n    df[\"prediction\"] = preds\n    maes = []\n    for t in df.type.unique():\n        y_true = df[df.type==t].scalar_coupling_constant.values\n        y_pred = df[df.type==t].prediction.values\n        mae = np.log(metrics.mean_absolute_error(y_true, y_pred))\n        maes.append(mae)\n    return np.mean(maes)\n\n#metric(test_, y_verif) #gets -inf\nmetric(test_, test_y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 4. Predicting and saving data"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_y = my_model.predict(np.array(X_test[['type','distance','distance_center0','distance_center1','distance_c0','distance_c1','distance_f0','distance_f1', 'type_0', 'type_1',\n                         'cos_f0', 'cos_f1',  'cos_c0_c1', 'cos_f0_f1','cos_c0', 'cos_c1', \"x_0\",\"y_0\",\"z_0\",\"x_1\",\"y_1\",\"z_1\",\"c_x\",\"c_y\",\"c_z\",\n                    'x_closest_0','y_closest_0','z_closest_0','x_closest_1','y_closest_1','z_closest_1']]))\nsub['scalar_coupling_constant'] = test_y\nsub.to_csv('submission.csv', index=False)\nsub.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}