{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom matplotlib import pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\nfrom sklearn.metrics import mean_absolute_error\nimport seaborn as sns\nimport lightgbm as lgb\nimport plotly\nimport plotly.graph_objs as go\nimport warnings\n\nfrom tabulate import tabulate\n\nwarnings.filterwarnings(\"ignore\")\n\nplotly.offline.init_notebook_mode(connected=False)\n\nimport os\nprint(os.listdir(\"../input\"))\n\ndata_dir = '../input/champs-scalar-coupling' if 'champs-scalar-coupling' in os.listdir('../input/') else '../input'","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(f'{data_dir}/train.csv')\ntest = pd.read_csv(f'{data_dir}/test.csv')\nsub = pd.read_csv(f'{data_dir}/sample_submission.csv')\nstructures = pd.read_csv(f'{data_dir}/structures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"train shape\", train.shape)\nprint(\"test shape\", train.shape)\nprint(\"structures shape\", structures.shape)\nprint(\"sub\", sub.shape)\nprint(\"train cols\", list(train.columns))\nprint(\"test cols\", list(test.columns))\nprint(\"structures cols\", list(structures.columns))\nprint(\"structures atoms\", list(np.unique(structures['atom'])))\nprint(\"\")\nprint(f\"There are {train['molecule_name'].nunique()} distinct molecules in train data.\")\nprint(f\"There are {test['molecule_name'].nunique()} distinct molecules in test data.\")\nprint(f\"There are {structures['atom'].nunique()} unique atoms in structures\")\nprint(f\"There are {train['type'].nunique()} unique types in train\")\nprint(f\"There are {test['type'].nunique()} unique types in test\")\nare_the_same_types = np.all(sorted(train['type'].unique()) == sorted(test['type'].unique()))\nprint(f\"Are all types in train and test the same? {are_the_same_types}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# merging\ntest['scalar_coupling_constant'] = np.nan\ntrain = train.merge(structures, left_on=['molecule_name', 'atom_index_0'], right_on=['molecule_name', 'atom_index'], how='left')\ntrain = train.merge(structures, left_on=['molecule_name', 'atom_index_1'], right_on=['molecule_name', 'atom_index'], how='left', suffixes=('_a', '_b'))\ntrain.drop(['atom_index_a', 'atom_index_b'], axis=1, inplace=True)\n\ntest = test.merge(structures, left_on=['molecule_name', 'atom_index_0'], right_on=['molecule_name', 'atom_index'], how='left')\ntest = test.merge(structures, left_on=['molecule_name', 'atom_index_1'], right_on=['molecule_name', 'atom_index'], how='left', suffixes=('_a', '_b'))\ntest.drop(['atom_index_a', 'atom_index_b'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('train shape', train.shape)\nprint('test shape', test.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Distribution of the target for each type\n\nAll the distributions of the target are very different if we split the data by type. The distribution of the target in the type 1JHN seems bimodal."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 4, figsize = (15, 7), sharex=True, sharey=True)\naxes = axes.flatten()\nfor i, type_ in enumerate(np.unique(train['type'])):\n    ix = train['type'] == type_\n    _ = sns.distplot(train['scalar_coupling_constant'][ix], ax=axes[i])\n    axes[i].set(title=f'{type_}. Rows: {ix.sum()}')\nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. ### Distribution of the target for each index_1 for type 1JHN\n\nSome differences can be seen if we split the data by atom_index_1 (plotting only the 8 most frequent ones)"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 4, figsize = (15, 5), sharex=True, sharey=False)\naxes = axes.flatten()\nsubset = train[train['type'] == '1JHN'].copy().reset_index()\nfor i, ai1 in enumerate(subset['atom_index_1'].value_counts().index[:8]):\n    ix = subset['atom_index_1'] == ai1\n    _ = sns.distplot(subset['scalar_coupling_constant'][ix], ax=axes[i])\n    axes[i].set(title=f'{ai1}. Rows: {ix.sum()}')\nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2D plot\n\nAgain, 1JHN seems to be a bit different from the rest of the types"},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, axes = plt.subplots(2, 4, figsize = (18, 8), sharex=True, sharey=True)\naxes = axes.flatten()\nfor i, type_ in enumerate(np.unique(train['type'])):\n    ix = train['type'] == type_\n    _ = sns.scatterplot(x='x_a', y='y_a', data=train.loc[ix].sample(20000), alpha=0.05, hue='scalar_coupling_constant', ax=axes[i])\n    axes[i].set(title=f'{type_}. Rows: {ix.sum()}')\nfig.tight_layout()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3D plot for x, y, z, and type\n A bit useless maybe, but cool.\n What is that structure that appears in the 3D space?"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(10)\nfrom plotly.graph_objs import FigureWidget\nlbl = LabelEncoder()\ntrain['type_id'] = lbl.fit_transform(train['type'])\n\ntraces = []\nfor i, type_ in enumerate(np.unique(train['type'])):\n    ix = train['type'] == type_\n    subset = train.loc[ix].sample(8000)\n    trace = go.Scatter3d(\n        name=type_,\n        x=subset['x_a'],\n        y=subset['y_a'],\n        z=subset['z_a'],\n        mode='markers',\n        marker=dict(\n            size=4,\n            opacity=0.03\n         )\n    )\n    traces.append(trace)\n\nlayout = go.Layout(\n    showlegend=True,\n    autosize=True,\n    scene=go.Scene(),\n    width=800,\n    height=1000,\n)\n\nFigureWidget(data=traces, layout=layout)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Colored by the target, ONLY TYPE 1JHN\n\nIn type 1JHN there seems to be clear clusters where the target value is higher or lower.\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"subset = train[train['type'] == '1JHN'].copy().reset_index().sample(43300)\n\ntraces = go.Scatter3d(\n    x=subset['x_a'],\n    y=subset['y_a'],\n    z=subset['z_a'],\n    mode='markers',\n    marker=dict(\n        size=4,\n        opacity=0.05,\n        color=subset['scalar_coupling_constant'],\n        colorscale='Viridis',\n     )\n)\n\nlayout = go.Layout(\n    autosize=True,\n    width=800,\n    height=1000,\n)\n\nFigureWidget(data=[traces], layout=layout)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Now the same for type 3JHN\n\nThe clusters are not that clear here..."},{"metadata":{"trusted":true},"cell_type":"code","source":"subset = train[train['type'] == '3JHN'].copy().reset_index().sample(43300)\ntraces = go.Scatter3d(\n    x=subset['x_a'],\n    y=subset['y_a'],\n    z=subset['z_a'],\n    mode='markers',\n    marker=dict(\n        size=4,\n        opacity=0.05,\n        color=subset['scalar_coupling_constant'],\n        colorscale='Viridis',\n     )\n)\n\nlayout = go.Layout(\n    autosize=True,\n    width=800,\n    height=1000,\n)\n\nFigureWidget(data=[traces], layout=layout)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}