{"cells":[{"metadata":{},"cell_type":"markdown","source":"This kernel is inspired by *rio114* kernel http://https://www.kaggle.com/rio114/coulomb-interaction\n> get distances from each atom belonging to the molecule and pickup 'num = 5' nearest regarding to each atom [H, C, N, O, F]\n\nI try to utilize pandas to calculate those interesting features instead of looping per molecule.\n\nIt speed up 56x faster from **216.16 s ** to **3.84 s** for first 100 molecules"},{"metadata":{},"cell_type":"markdown","source":"## Load the data"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport time\nimport datetime as dt\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.neural_network import MLPRegressor\n\nimport warnings\nwarnings.simplefilter('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"FOLDER = '../input/'\nOUTPUT = '../preprocessed/'\nos.listdir(FOLDER)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_structures = pd.read_csv(os.path.join(FOLDER, 'structures.csv'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compute Distance"},{"metadata":{},"cell_type":"markdown","source":"I use self join instead of distance matrix to get the distance of atoms among molecule because I want to avoid looping per molecule\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_distance = df_structures.merge(df_structures, how = 'left', on= 'molecule_name', suffixes = ('_0', '_1'))\n# remove same molecule\ndf_distance = df_distance.loc[df_distance['atom_index_0'] != df_distance['atom_index_1']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_distance['distance'] = np.linalg.norm(df_distance[['x_0','y_0', 'z_0']].values - \n                                           df_distance[['x_1', 'y_1', 'z_1']].values, axis=1, ord = 2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_distance.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can use ```df_distance``` to join to train and test dataset "},{"metadata":{},"cell_type":"markdown","source":"## Interaction Data Frame"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_interaction_data_frame(df_distance, num_nearest = 5):\n    time_start = dt.datetime.now()\n    print(\"START\", time_start)\n    \n    # get nearest 5 (num_nearest) by distances\n    df_temp = df_distance.groupby(['molecule_name', 'atom_index_0', 'atom_1'])['distance'].nsmallest(num_nearest)\n    \n    # make it clean\n    df_temp = pd.DataFrame(df_temp).reset_index()[['molecule_name', 'atom_index_0', 'atom_1', 'distance']]\n    df_temp.columns = ['molecule_name', 'atom_index', 'atom', 'distance']\n    \n    time_nearest = dt.datetime.now()\n    print(\"Time Nearest\", time_nearest-time_start)\n    \n    # get rank by distance\n    df_temp['distance_rank'] = df_temp.groupby(['molecule_name', 'atom_index', 'atom'])['distance'].rank(ascending = True, method = 'first').astype(int)\n    \n    time_rank = dt.datetime.now()\n    print(\"Time Rank\", time_rank-time_nearest)\n    \n    # pivot to get nearest distance by atom type \n    df_distance_nearest = pd.pivot_table(df_temp, index = ['molecule_name','atom_index'], columns= ['atom', 'distance_rank'], values= 'distance')\n    \n    time_pivot = dt.datetime.now()\n    print(\"Time Pivot\", time_pivot-time_rank)\n    del df_temp\n    \n    columns_distance_nearest =  np.core.defchararray.add('distance_nearest_', \n                                          np.array(df_distance_nearest.columns.get_level_values('distance_rank')).astype(str) +  \n                                          np.array(df_distance_nearest.columns.get_level_values('atom')) )\n    df_distance_nearest.columns = columns_distance_nearest\n    \n    # 1 / r^2 to get the square inverse same with the previous kernel\n    df_distance_sq_inv_farthest = 1 / (df_distance_nearest ** 2)\n    \n    columns_distance_sq_inv_farthest = [col.replace('distance_nearest', 'distance_sq_inv_farthest') for col in columns_distance_nearest]\n\n    df_distance_sq_inv_farthest.columns = columns_distance_sq_inv_farthest\n    time_inverse = dt.datetime.now()\n    print(\"Time Inverse Calculation\", time_inverse-time_pivot)\n    \n    df_interaction = pd.concat([df_distance_sq_inv_farthest, df_distance_nearest] , axis = 1)\n    df_interaction.reset_index(inplace = True)\n    \n    time_concat = dt.datetime.now()\n    print(\"Time Concat\", time_concat-time_inverse)\n    \n    return df_interaction","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"first_100_molecules = df_structures['molecule_name'].unique()[:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_interaction = get_interaction_data_frame(df_distance.loc[df_distance['molecule_name'].isin(first_100_molecules)])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Computing **N smallest object** is take a lot of time compare to others step. \n\nWe need to tackle that compuation, because it still to long if we want to run for all molecule"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_interaction.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"The result is same with http://https://www.kaggle.com/rio114/coulomb-interaction \njust need to join to train dataset"},{"metadata":{},"cell_type":"markdown","source":"Hope This is helps for our feature engineering\n\nCheers :D  "}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}