{"cells":[{"metadata":{},"cell_type":"markdown","source":"EDA for Molecule in this kernel, including 3D analysis, and also raised a problem about the train and test dataset, looking for someone's help, or team up.\nLack of chemical knowledge.\nMy first public kernel, if it helps, please upvote it, thanks."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/test.csv')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures = pd.read_csv('../input/structures.csv')\nstructures.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# For all"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape, test.shape, structures.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are {} unique molecules in trainset'.format(train['molecule_name'].nunique()))\nprint('There are {} unique molecules in testset'.format(test['molecule_name'].nunique()))\nprint('There are {} unique molecules in structures'.format(structures['molecule_name'].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There are {} atom_index_0 & {} atom_index_1 & {} types in trainset'.format(train['atom_index_0'].nunique(), train['atom_index_1'].nunique(), train['type'].nunique()))\nprint('There are {} atom_index_0 & {} atom_index_1 & {} types in testset'.format(test['atom_index_0'].nunique(), test['atom_index_1'].nunique(), test['type'].nunique()))\nprint('There are {} atom_index & {} atom in structures'.format(structures['atom_index'].nunique(), structures['atom'].nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\nfrom matplotlib import pyplot as plt\n\nplt.subplots(figsize=(24, 6))\nplt.subplot(1, 3, 1)\nplt.title(\"train['atom_index_0']\")\nplt.hist(train['atom_index_0'], bins=29)\nplt.subplot(1, 3, 2)\nplt.title(\"train['atom_index_1']\")\nplt.hist(train['atom_index_1'], bins=29)\nplt.subplot(1, 3, 3)\nplt.title(\"train['scalar_coupling_constant']\")\nplt.hist(train['scalar_coupling_constant'], bins=100)\nplt.show()\n\nplt.subplots(figsize=(16, 6))\nplt.subplot(1, 2, 1)\nplt.title(\"test['atom_index_0']\")\nplt.hist(test['atom_index_0'], bins=29)\nplt.subplot(1, 2, 2)\nplt.title(\"test['atom_index_1']\")\nplt.hist(test['atom_index_1'], bins=29)\nplt.show()\n\nplt.subplots(figsize=(24, 6))\nplt.subplot(1, 3, 1)\nplt.title(\"structures['x']\")\nplt.hist(structures['x'], bins=100)\nplt.subplot(1, 3, 2)\nplt.title(\"structures['y']\")\nplt.hist(structures['y'], bins=100)\nplt.subplot(1, 3, 3)\nplt.title(\"structures['z']\")\nplt.hist(structures['z'], bins=100)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_atom = pd.concat([train['atom_index_0'], train['atom_index_1']])\ntest_atom = pd.concat([test['atom_index_0'], test['atom_index_1']])\nplt.subplots(figsize=(16, 6))\nplt.subplot(1, 2, 1)\nplt.title(\"train_atom\")\nplt.hist(train_atom, bins=29)\nplt.subplot(1, 2, 2)\nplt.title(\"test_atom\")\nplt.hist(test_atom, bins=29)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# For each type"},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nfig, axes = plt.subplots(2, 4, figsize = (18, 8))\naxes = axes.flatten()\nfor i, type_ in enumerate(np.unique(train['type'])):\n    ix = train['type'] == type_\n    _ = sns.distplot(train['scalar_coupling_constant'][ix], ax=axes[i])\n    axes[i].set(title=f'{type_}. Rows: {ix.sum()}')\nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['scalar_coupling_constant'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1JHC = train[train['type'] == '1JHC']\ntrain_1JHN = train[train['type'] == '1JHN']\ntrain_2JHC = train[train['type'] == '2JHC']\ntrain_2JHH = train[train['type'] == '2JHH']\ntrain_2JHN = train[train['type'] == '2JHN']\ntrain_3JHC = train[train['type'] == '3JHC']\ntrain_3JHH = train[train['type'] == '3JHH']\ntrain_3JHN = train[train['type'] == '2JHN']\n\nprint('train_1JHC atom unique', pd.concat([train_1JHC['atom_index_0'], train_1JHC['atom_index_1']]).nunique())\nprint('train_1JHN atom unique', pd.concat([train_1JHN['atom_index_0'], train_1JHN['atom_index_1']]).nunique())\nprint('train_2JHC atom unique', pd.concat([train_2JHC['atom_index_0'], train_2JHC['atom_index_1']]).nunique())\nprint('train_2JHH atom unique', pd.concat([train_2JHH['atom_index_0'], train_2JHH['atom_index_1']]).nunique())\nprint('train_2JHN atom unique', pd.concat([train_2JHN['atom_index_0'], train_2JHN['atom_index_1']]).nunique())\nprint('train_3JHC atom unique', pd.concat([train_3JHC['atom_index_0'], train_3JHC['atom_index_1']]).nunique())\nprint('train_3JHH atom unique', pd.concat([train_3JHH['atom_index_0'], train_3JHH['atom_index_1']]).nunique())\nprint('train_3JHN atom unique', pd.concat([train_3JHN['atom_index_0'], train_3JHN['atom_index_1']]).nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_1JHC = test[test['type'] == '1JHC']\ntest_1JHN = test[test['type'] == '1JHN']\ntest_2JHC = test[test['type'] == '2JHC']\ntest_2JHH = test[test['type'] == '2JHH']\ntest_2JHN = test[test['type'] == '2JHN']\ntest_3JHC = test[test['type'] == '3JHC']\ntest_3JHH = test[test['type'] == '3JHH']\ntest_3JHN = test[test['type'] == '2JHN']\n\nprint('test_1JHC atom unique', pd.concat([test_1JHC['atom_index_0'], test_1JHC['atom_index_1']]).nunique())\nprint('test_1JHN atom unique', pd.concat([test_1JHN['atom_index_0'], test_1JHN['atom_index_1']]).nunique())\nprint('test_2JHC atom unique', pd.concat([test_2JHC['atom_index_0'], test_2JHC['atom_index_1']]).nunique())\nprint('test_2JHH atom unique', pd.concat([test_2JHH['atom_index_0'], test_2JHH['atom_index_1']]).nunique())\nprint('test_2JHN atom unique', pd.concat([test_2JHN['atom_index_0'], test_2JHN['atom_index_1']]).nunique())\nprint('test_3JHC atom unique', pd.concat([test_3JHC['atom_index_0'], test_3JHC['atom_index_1']]).nunique())\nprint('test_3JHH atom unique', pd.concat([test_3JHH['atom_index_0'], test_3JHH['atom_index_1']]).nunique())\nprint('test_3JHN atom unique', pd.concat([test_3JHN['atom_index_0'], test_3JHN['atom_index_1']]).nunique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplots(8, 2, figsize=(12, 24))\nplt.subplot(8, 2, 1)\nplt.hist(pd.concat([train_1JHC['atom_index_0'], train_1JHC['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 2)\nplt.hist(pd.concat([test_1JHC['atom_index_0'], test_1JHC['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 3)\nplt.hist(pd.concat([train_1JHN['atom_index_0'], train_1JHN['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 4)\nplt.hist(pd.concat([test_1JHN['atom_index_0'], test_1JHN['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 5)\nplt.hist(pd.concat([train_2JHC['atom_index_0'], train_2JHC['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 6)\nplt.hist(pd.concat([test_2JHC['atom_index_0'], test_2JHC['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 7)\nplt.hist(pd.concat([train_2JHH['atom_index_0'], train_2JHH['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 8)\nplt.hist(pd.concat([test_2JHH['atom_index_0'], test_2JHH['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 9)\nplt.hist(pd.concat([train_2JHN['atom_index_0'], train_2JHN['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 10)\nplt.hist(pd.concat([test_2JHN['atom_index_0'], test_2JHN['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 11)\nplt.hist(pd.concat([train_3JHC['atom_index_0'], train_3JHC['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 12)\nplt.hist(pd.concat([test_3JHC['atom_index_0'], test_3JHC['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 13)\nplt.hist(pd.concat([train_3JHH['atom_index_0'], train_3JHH['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 14)\nplt.hist(pd.concat([test_3JHH['atom_index_0'], test_3JHH['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 15)\nplt.hist(pd.concat([train_3JHN['atom_index_0'], train_3JHN['atom_index_1']]), bins=29)\nplt.subplot(8, 2, 16)\nplt.hist(pd.concat([test_3JHN['atom_index_0'], test_3JHN['atom_index_1']]), bins=29)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<font size=5>Problem here.</font>"},{"metadata":{},"cell_type":"markdown","source":"<font size=3>Here is the problem, unbalanced data for 3JHH between train and test.\nIf anywhere of my code is wrong? Or is it the truth?\nHow to deal with that?</font>"},{"metadata":{},"cell_type":"markdown","source":"# For structure"},{"metadata":{},"cell_type":"markdown","source":"Reference: https://www.kaggle.com/chechir/molecular-eda-3d-plots"},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.merge(structures, left_on=['molecule_name', 'atom_index_0'], right_on=['molecule_name', 'atom_index'], how='left')\ntrain = train.merge(structures, left_on=['molecule_name', 'atom_index_1'], right_on=['molecule_name', 'atom_index'], how='left', suffixes=('_a', '_b'))\ntrain.drop(['atom_index_0', 'atom_index_1'], axis=1, inplace=True)\n\ntest['scalar_coupling_constant'] = np.nan\ntest = test.merge(structures, left_on=['molecule_name', 'atom_index_0'], right_on=['molecule_name', 'atom_index'], how='left')\ntest = test.merge(structures, left_on=['molecule_name', 'atom_index_1'], right_on=['molecule_name', 'atom_index'], how='left', suffixes=('_a', '_b'))\ntest.drop(['atom_index_0', 'atom_index_1'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train set shape: {}, test set shape: {}'.format(train.shape, test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train atom_0 types: {}, train atom_1 types: {}'.format(train['atom_a'].nunique(), train['atom_b'].nunique()))\nprint('test atom_0 types: {}, test atom_1 types: {}'.format(test['atom_a'].nunique(), test['atom_b'].nunique()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3D analysis"},{"metadata":{},"cell_type":"markdown","source":"Reference: https://www.kaggle.com/pestipeti/interactive-3d-molecule-structure"},{"metadata":{"trusted":true},"cell_type":"code","source":"import plotly.offline as py\nimport plotly.graph_objs as go\n\nfrom plotly import tools\n\ncolor = sns.color_palette()\n%matplotlib inline\npy.init_notebook_mode(connected=True)\n\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\ndef show_molecule(mdata, mstruct):\n    mdata = mdata.merge(right=mstruct, how='left',\n                        left_on=['molecule_name', 'atom_index_0'],\n                        right_on=['molecule_name', 'atom_index'])\n    mdata.rename(index=str, columns={\"x\": \"x0\", \"y\": \"y0\", \"z\": \"z0\", \"atom\": \"atom0\"}, inplace=True)\n    mdata.drop(['atom_index'], axis=1, inplace=True)\n\n    mdata = mdata.merge(right=mstruct, how='left',\n                  left_on=['molecule_name', 'atom_index_1'],\n                  right_on=['molecule_name', 'atom_index']\n                 )\n    mdata.rename(index=str, columns={\"x\": \"x1\", \"y\": \"y1\", \"z\": \"z1\", \"atom\": \"atom1\"}, inplace=True)\n    mdata.drop(['atom_index'], axis=1, inplace=True)    \n    \n    data = []\n  \n    atoms = mstruct['atom'].unique()\n    types = mdata['type'].unique()\n    \n    atom_cfg = {\n        'H': {\"name\": \"Hydrogen\", \"color\": \"#757575\", \"size\": 4},\n        'C': {\"name\": \"Carbon\", \"color\": \"#f44336\", \"size\": 12},\n        'O': {\"name\": \"Oxygen\", \"color\": \"#03a9f4\", \"size\": 12},\n        'N': {\"name\": \"Nitrogen\", \"color\": \"#ff9800\", \"size\": 12},\n        'F': {\"name\": \"Fluorine\", \"color\": \"#673ab7\", \"size\": 12},\n    }\n    \n    type_cfg = {\n        '2JHH': {\"color\": \"#757575\", \"width\": 2},\n        '3JHH': {\"color\": \"#757575\", \"width\": 3},\n\n        '1JHC': {\"color\": \"#f44336\", \"width\": 1},\n        '2JHC': {\"color\": \"#f44336\", \"width\": 2},\n        '3JHC': {\"color\": \"#f44336\", \"width\": 3},\n\n        '1JHN': {\"color\": \"#ff9800\", \"width\": 2},\n        '2JHN': {\"color\": \"#ff9800\", \"width\": 2},\n        '3JHN': {\"color\": \"#ff9800\", \"width\": 3},\n    }\n\n    for atom, config in atom_cfg.items(): \n        if atom in atoms:\n            data.append(\n                go.Scatter3d(\n                    x=mstruct[mstruct['atom'] == atom]['x'].values,\n                    y=mstruct[mstruct['atom'] == atom]['y'].values,\n                    z=mstruct[mstruct['atom'] == atom]['z'].values,\n                    mode='markers',\n                    marker=dict(\n                        color=config['color'],\n                        size=config['size'],\n                        opacity=0.8\n                    ),\n                    name=config['name']\n                )\n            )\n\n    for ctype, config in type_cfg.items():\n        if ctype in types:\n            eX = []; eY = []; eZ = []\n            for row in mdata[mdata['type'] == ctype].iterrows():\n                rd = row[1]\n                eX += [rd['x0'], rd['x1']]\n                eY += [rd['y0'], rd['y1']]\n                eZ += [rd['z0'], rd['z1']]            \n            \n            data.append(\n                go.Scatter3d(\n                    x=eX,\n                    y=eY,\n                    z=eZ,\n                    mode='lines',\n                    line=dict(color=config['color'], width=config['width']),\n                    name=ctype\n                )\n            )            \n\n    axis=dict(showbackground=True, showline=False, zeroline=False, showgrid=True, showticklabels=False, title='')\n    layout = go.Layout(\n        margin=dict(l=50, r=50, b=50, t=50),\n        width=720,\n        height=640,\n        showlegend=True,\n        scene=dict(\n            xaxis=dict(axis),\n            yaxis=dict(axis),\n            zaxis=dict(axis),\n        )\n    )\n    fig = go.Figure(data=data, layout=layout)\n    py.iplot(fig, filename='molecule')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures = pd.read_csv('../input/structures.csv')\ndef show_train(name, train):\n    print(name)\n#     molecule = name\n#     print(train[train['molecule_name'] == molecule])\n    show_molecule(train[train['molecule_name'] == name], structures[structures['molecule_name'] == name])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntest['scalar_coupling_constant'] = np.nan\nmolecules = pd.concat([train, test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(1, 5):\n    show_train('dsgdb9nsd_00000'+str(i), molecules)\n    show_train('dsgdb9nsd_00001'+str(i), molecules)\n    show_train('dsgdb9nsd_00011'+str(i), molecules)\n    show_train('dsgdb9nsd_10000'+str(i), molecules)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"molecule = 'dsgdb9nsd_128739'\nshow_molecule(molecules[molecules['molecule_name'] == molecule], structures[structures['molecule_name'] == molecule])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}