{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.preprocessing import OneHotEncoder\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading train and test data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/predicting-molecular-properties-datamerging/train.csv\")\ntest = pd.read_csv(\"../input/predicting-molecular-properties-datamerging/test.csv\")\nstructures = pd.read_csv(\"../input/champs-scalar-coupling/structures.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Droping columns which can not be used for this prediction since those data is not available for test dataset."},{"metadata":{"trusted":true},"cell_type":"code","source":"train.drop(['X', 'Y', 'Z', 'potential_energy','XX_atom_0', 'YX_atom_0', 'ZX_atom_0', 'XY_atom_0', 'YY_atom_0',\n       'ZY_atom_0', 'XZ_atom_0', 'YZ_atom_0', 'ZZ_atom_0',\n       'mulliken_charge_atom_0','XX_atom_1', 'YX_atom_1', 'ZX_atom_1',\n       'XY_atom_1', 'YY_atom_1', 'ZY_atom_1', 'XZ_atom_1', 'YZ_atom_1',\n       'ZZ_atom_1', 'mulliken_charge_atom_1'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating total atoms in each molecule."},{"metadata":{"trusted":true},"cell_type":"code","source":"total_atoms_per_molecule = structures[['molecule_name']]\ntotal_atoms_per_molecule['atoms_in_molecule'] = 1\ntotal_atoms_per_molecule = total_atoms_per_molecule.groupby('molecule_name').sum()\ntotal_atoms_per_molecule.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating element wise atoms in each molecule."},{"metadata":{"trusted":true},"cell_type":"code","source":"element_wise_atoms_per_molecule = structures[['molecule_name', 'atom']]\nelement_wise_atoms_per_molecule['count'] = 1\nelement_wise_atoms_per_molecule = element_wise_atoms_per_molecule.groupby(['molecule_name', 'atom']).sum()\n\natom_table = pd.pivot_table(element_wise_atoms_per_molecule, values='count', index=['molecule_name'],\n                      columns=['atom']).fillna(0)\natom_table.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating coupling type wise number of pairs in each molecule."},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cout_by_type(data):\n    type_wise_bonds_per_molecule = data[['molecule_name', 'type']]\n    type_wise_bonds_per_molecule['count'] = 1\n    type_wise_bonds_per_molecule = type_wise_bonds_per_molecule.groupby(['molecule_name', 'type']).sum()   \n    type_table = pd.pivot_table(type_wise_bonds_per_molecule, values='count', index=['molecule_name'],\n                                columns=['type']).fillna(0)\n    return type_table","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Calculating distance between 2 atoms"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_distance(data):\n    data['x_dist'] = abs(data['x_atom_0'] - data['x_atom_1'])\n    data['y_dist'] = abs(data['y_atom_0'] - data['y_atom_1'])\n    data['z_dist'] = abs(data['z_atom_0'] - data['z_atom_1'])\n    data['2_atom_dist'] = np.sqrt(data['x_dist']**2 + data['y_dist']**2 + data['z_dist']**2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calculate_distance(train)\ncalculate_distance(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_mean_dist_mol_type(data):\n    mean_dist_molecule_type = data[['molecule_name', 'type', '2_atom_dist']]\n    mean_dist_molecule_type = mean_dist_molecule_type.groupby(['molecule_name', 'type']).mean()\n    mean_dist_molecule_type.columns = ['mean_dist_for_mol_type']\n#     print(mean_dist_molecule.head(20))\n    return mean_dist_molecule_type","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_mean_dist_mol(data):\n    mean_dist_molecule = data[['molecule_name', '2_atom_dist']]\n    mean_dist_molecule = mean_dist_molecule.groupby(['molecule_name']).mean()\n    mean_dist_molecule.columns = ['mean_dist_for_mol']\n#     print(mean_dist_molecule.head(20))\n    return mean_dist_molecule","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_mean_dist_type(data):\n    mean_dist_type = data[['type', '2_atom_dist']]\n    mean_dist_type = mean_dist_type.groupby(['type']).mean()\n    mean_dist_type.columns = ['mean_dist_for_type']\n#     print(mean_dist_type.head(20))\n    return mean_dist_type","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_max_dist_mol_type(data):\n    max_dist_molecule_type = data[['molecule_name', 'type', '2_atom_dist']]\n    max_dist_molecule_type = max_dist_molecule_type.groupby(['molecule_name', 'type']).max()\n    max_dist_molecule_type.columns = ['max_dist_for_mol_type']\n#     print(max_dist_molecule.head(20))\n    return max_dist_molecule_type","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_max_dist_mol(data):\n    max_dist_molecule = data[['molecule_name', '2_atom_dist']]\n    max_dist_molecule = max_dist_molecule.groupby(['molecule_name']).max()\n    max_dist_molecule.columns = ['max_dist_for_mol']\n#     print(max_dist_molecule.head(20))\n    return max_dist_molecule","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_max_dist_type(data):\n    max_dist_type = data[['type', '2_atom_dist']]\n    max_dist_type = max_dist_type.groupby(['type']).max()\n    max_dist_type.columns = ['max_dist_for_type']\n#     print(max_dist_type.head(20))\n    return max_dist_type","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_min_dist_mol_type(data):\n    min_dist_molecule_type = data[['molecule_name', 'type', '2_atom_dist']]\n    min_dist_molecule_type = min_dist_molecule_type.groupby(['molecule_name', 'type']).min()\n    min_dist_molecule_type.columns = ['min_dist_for_mol_type']\n#     print(min_dist_molecule.head(20))\n    return min_dist_molecule_type","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_min_dist_mol(data):\n    min_dist_molecule = data[['molecule_name', '2_atom_dist']]\n    min_dist_molecule = min_dist_molecule.groupby(['molecule_name']).min()\n    min_dist_molecule.columns = ['min_dist_for_mol']\n#     print(min_dist_molecule.head(20))\n    return min_dist_molecule","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_min_dist_type(data):\n    min_dist_type = data[['type', '2_atom_dist']]\n    min_dist_type = min_dist_type.groupby(['type']).min()\n    min_dist_type.columns = ['min_dist_for_type']\n#     print(min_dist_type.head(20))\n    return min_dist_type","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(total_atoms_per_molecule, on='molecule_name')\ntrain = train.merge(atom_table, on='molecule_name')\ntrain = train.merge(get_cout_by_type(train), on='molecule_name')\n\ntrain = train.merge(calculate_mean_dist_mol(train), on='molecule_name')\ntrain = train.merge(calculate_max_dist_mol(train), on='molecule_name')\ntrain = train.merge(calculate_min_dist_mol(train), on='molecule_name')\n\ntrain = train.merge(calculate_mean_dist_type(train), on='type', how='left')\ntrain = train.merge(calculate_max_dist_type(train), on='type', how='left')\ntrain = train.merge(calculate_min_dist_type(train), on='type', how='left')\n\ntrain = train.merge(calculate_mean_dist_mol_type(train), on=['molecule_name', 'type'])\ntrain = train.merge(calculate_max_dist_mol_type(train), on=['molecule_name', 'type'])\ntrain = train.merge(calculate_min_dist_mol_type(train), on=['molecule_name', 'type'])\n\ntest = test.merge(total_atoms_per_molecule, on='molecule_name')\ntest = test.merge(atom_table, on='molecule_name')\ntest = test.merge(get_cout_by_type(test), on='molecule_name')\n\ntest = test.merge(calculate_mean_dist_mol(test), on='molecule_name')\ntest = test.merge(calculate_max_dist_mol(test), on='molecule_name')\ntest = test.merge(calculate_min_dist_mol(test), on='molecule_name')\n\ntest = test.merge(calculate_mean_dist_type(test), on='type', how='left')\ntest = test.merge(calculate_max_dist_type(test), on='type', how='left')\ntest = test.merge(calculate_min_dist_type(test), on='type', how='left')\n\ntest = test.merge(calculate_mean_dist_mol_type(test), on=['molecule_name', 'type'])\ntest = test.merge(calculate_max_dist_mol_type(test), on=['molecule_name', 'type'])\ntest = test.merge(calculate_min_dist_mol_type(test), on=['molecule_name', 'type'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_features(data):\n    data['dist-mean_dist_mol'] = data['2_atom_dist'] - data['mean_dist_for_mol']\n    data['dist-min_dist_mol'] = data['2_atom_dist'] - data['min_dist_for_mol']\n    data['max_dist_mol-dist'] = data['max_dist_for_mol'] - data['2_atom_dist']\n    \n    data['dist-mean_dist_type'] = data['2_atom_dist'] - data['mean_dist_for_type']\n    data['dist-min_dist_type'] = data['2_atom_dist'] - data['min_dist_for_type']\n    data['max_dist_type-dist'] = data['max_dist_for_type'] - data['2_atom_dist']\n    \n    data['dist-mean_dist_mol_type'] = data['2_atom_dist'] - data['mean_dist_for_mol_type']\n    data['dist-min_dist_mol_type'] = data['2_atom_dist'] - data['min_dist_for_mol_type']\n    data['max_dist_mol-dist_type'] = data['max_dist_for_mol_type'] - data['2_atom_dist']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"add_features(train)\nadd_features(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"types = train[['molecule_name', 'type']]\ntypes = types.groupby('type').count().sort_values('molecule_name', ascending=False)\n\nplt.figure(figsize=(10,8))\nsns.barplot(types.index, types['molecule_name'])\nplt.title('Coupling type wise number of pairs')\nplt.ylabel('Number of pairs')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# coupling = train[['type', 'scalar_coupling_constant']]\n# plt.figure(figsize=(20,12))\n# sns.violinplot(x='type', y='scalar_coupling_constant', data=coupling)\n# plt.ylabel('scalar_coupling_constant')\n# plt.title('Distribution of scalar coupling constant')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Scatter plot of Distance between 2 atoms vs scalar coupling constant."},{"metadata":{"trusted":true},"cell_type":"code","source":"# plt.figure(figsize=(20,12))\n# sns.relplot(x='2_atom_dist', y='scalar_coupling_constant', hue='type', data=train)\n# plt.xlabel('Distance between 2 atoms')\n# plt.ylabel('scalar coupling constant')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Average distance vs average scalar coupling constant. Here size of the marker shows the number of pairs for that coupling type."},{"metadata":{"trusted":true},"cell_type":"code","source":"# avg_dist = train[['type', '2_atom_dist', 'scalar_coupling_constant']]\n# avg_dist['count'] = 1\n# avg_dist = avg_dist.groupby('type').agg({'2_atom_dist': np.mean,\n#                                        'scalar_coupling_constant': np.mean,\n#                                        'count':np.sum})\n# sns.relplot(x='2_atom_dist', y='scalar_coupling_constant', hue=avg_dist.index, \n#             size='count', sizes=(50,1000), data=avg_dist)\n# plt.xlabel('Avg. distance between 2 atoms')\n# plt.ylabel('Avg. scalar coupling constant')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"columns = ['type',\n           '2_atom_dist', 'dist-mean_dist_mol',\n       'dist-min_dist_mol', 'max_dist_mol-dist', 'dist-mean_dist_type',\n       'dist-min_dist_type', 'max_dist_type-dist',\n           'fc', 'sd', 'pso', 'dso',\n           'scalar_coupling_constant']\n\ndf_1 = train[columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def draw_heatmap(df):\n    corr_mx = df.corr()\n    plt.figure(figsize=(12,8))\n    sns.heatmap(corr_mx, annot=True, vmin=-1, vmax=1, center=0)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '1JHC']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '1JHN']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '2JHC']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '2JHN']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '2JHH']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '3JHC']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '3JHN']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df_1[df_1['type'] == '3JHH']\ndraw_heatmap(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def to_onehot_type(data):\n    encoder = OneHotEncoder(sparse=False)\n    onehot_type = encoder.fit_transform(data['type'].values.reshape(len(data),1))\n    df_onehot = pd.DataFrame(onehot_type, columns=['type_'+t for t in encoder.categories_[0]])\n    data = pd.concat([data, df_onehot], axis=1)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = to_onehot_type(train)\ntest = to_onehot_type(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train[['id', 'type',\n               'type_1JHC', 'type_1JHN', 'type_2JHC', 'type_2JHH', \n               'type_2JHN', 'type_3JHC', 'type_3JHH', 'type_3JHN', \n               'atoms_in_molecule', \n               'C', 'F', 'H', 'N', 'O', \n               '1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN',\n               '2_atom_dist', \n               'mean_dist_for_mol', 'max_dist_for_mol', 'min_dist_for_mol', \n               'mean_dist_for_type', 'max_dist_for_type', 'min_dist_for_type', \n               'dist-mean_dist_mol', 'dist-min_dist_mol', 'max_dist_mol-dist', \n               'dist-mean_dist_type', 'dist-min_dist_type', 'max_dist_type-dist',\n               'mean_dist_for_mol_type', 'max_dist_for_mol_type', 'min_dist_for_mol_type',\n               'dist-mean_dist_mol_type', 'dist-min_dist_mol_type', 'max_dist_mol-dist_type',\n               'fc', 'sd', 'pso', 'dso', \n               'scalar_coupling_constant']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = test[['id', 'type',\n             'type_1JHC', 'type_1JHN', 'type_2JHC', 'type_2JHH', \n             'type_2JHN', 'type_3JHC', 'type_3JHH', 'type_3JHN', \n             'atoms_in_molecule', \n             'C', 'F', 'H', 'N', 'O', \n             '1JHC', '1JHN', '2JHC', '2JHH', '2JHN', '3JHC', '3JHH', '3JHN',\n             '2_atom_dist', \n             'mean_dist_for_mol', 'max_dist_for_mol', 'min_dist_for_mol', \n             'mean_dist_for_type', 'max_dist_for_type', 'min_dist_for_type', \n             'dist-mean_dist_mol', 'dist-min_dist_mol', 'max_dist_mol-dist', \n             'dist-mean_dist_type', 'dist-min_dist_type', 'max_dist_type-dist',\n             'mean_dist_for_mol_type', 'max_dist_for_mol_type', 'min_dist_for_mol_type',\n             'dist-mean_dist_mol_type', 'dist-min_dist_mol_type', 'max_dist_mol-dist_type']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.to_csv('train.csv', index=False)\ntest.to_csv('test.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}