{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction\n\nIn this notebook we are going to create 3 different files ready to perform EDA in the **fastest way**:\n\n - **train.msg** : Contains the train.csv data + structures + distance metric\n - **test.msg** : Contains the test.csv data + structures + distance metric\n - **complete.msg** : Contains the train.csv + all the data from the complementary files + distance metric\n \nThe files are saved in [msgpack format](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_msgpack.html). This allow us to save/load the dataframes very fast (and the dtypes are saved too, so no conversion is needed when load).\n\nSome people are afraid to use msgpack, because in the pandas docs it is stated that the function is experimental. After one year using mspack, I can ensure that no problem has arrise to me. Anyway you can use csv if you like it more.\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom IPython.display import display\n\ndata_path = '../input'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls -lSh $data_path/*.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"files_names = !ls $data_path/*.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dict = {}\n\nfor name in files_names:\n    data_dict[name.split('/')[-1][:-4]] = pd.read_csv(name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Lets see how the files looks like"},{"metadata":{"trusted":true},"cell_type":"code","source":"for k in data_dict.keys():\n    display(k)\n    display(data_dict[k].head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# lets join this data in single dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_complete = data_dict['train'].copy()\ndf_complete = df_complete.join(data_dict['potential_energy'].set_index('molecule_name'), on='molecule_name')\ndf_complete = df_complete.join(data_dict['dipole_moments'].set_index('molecule_name'), on='molecule_name', lsuffix='dipole_moments_')\ndf_complete = df_complete.join(data_dict['magnetic_shielding_tensors'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_0'], lsuffix='_atom0')\ndf_complete = df_complete.join(data_dict['magnetic_shielding_tensors'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_1'], lsuffix='_atom1')\ndf_complete = df_complete.join(data_dict['mulliken_charges'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_0'], lsuffix='_atom0')\ndf_complete = df_complete.join(data_dict['mulliken_charges'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_1'], lsuffix='_atom1')\ndf_complete = df_complete.join(data_dict['scalar_coupling_contributions'].set_index(['molecule_name', 'atom_index_0', 'atom_index_1']), on=['molecule_name', 'atom_index_0', 'atom_index_1'], rsuffix='_scc')\ndf_complete = df_complete.join(data_dict['structures'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_0'], lsuffix='_atom0_structure')\ndf_complete = df_complete.join(data_dict['structures'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_1'], lsuffix='_atom1_structure')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_train = data_dict['train'].copy()\ndf_train = df_train.join(data_dict['structures'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_0'], lsuffix='_atom0_structure')\ndf_train = df_train.join(data_dict['structures'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_1'], lsuffix='_atom1_structure')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndf_test = data_dict['test'].copy()\ndf_test = df_test.join(data_dict['structures'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_0'], lsuffix='_atom0_structure')\ndf_test = df_test.join(data_dict['structures'].set_index(['molecule_name', 'atom_index']), on=['molecule_name', 'atom_index_1'], lsuffix='_atom1_structure')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# calculate distance"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfor df in [df_complete, df_train, df_test]:    \n    distance_foo = np.linalg.norm(df[['x_atom1_structure', 'y_atom1_structure', 'z_atom1_structure']].values - df[['x', 'y', 'z']].values, axis=1)\n    df['distance'] = distance_foo","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# save the data "},{"metadata":{"trusted":false},"cell_type":"code","source":"df_complete.to_msgpack('./complete.msg')\ndf_train.to_msgpack('./train.msg')\ndf_test.to_msgpack('./test.msg')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Have a nice EDA ;)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}