{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Load Everything"},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nimport pandas as pd\nimport numpy as np\n\nimport math\nimport gc\nimport copy\n\nfrom sklearn.model_selection import KFold, train_test_split\nfrom sklearn.metrics import mean_absolute_error\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nfrom xgboost import XGBRegressor\nimport os\nfrom sklearn.decomposition import PCA","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = '../input'\nSUBMISSIONS_PATH = './'\n# use atomic numbers to recode atomic names\nATOMIC_NUMBERS = {\n    'H': 1,\n    'C': 6,\n    'N': 7,\n    'O': 8,\n    'F': 9\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option('display.max_colwidth', -1)\npd.set_option('display.max_rows', 120)\npd.set_option('display.max_columns', 120)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load Dataset"},{"metadata":{},"cell_type":"markdown","source":"By default all data is read as `float64` and `int64`. We can trade this uneeded precision for memory and higher prediction speed. So, let's read with Pandas all the data in the minimal representation: "},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dtypes = {\n    'molecule_name': 'category',\n    'atom_index_0': 'int8',\n    'atom_index_1': 'int8',\n    'type': 'category',\n    'scalar_coupling_constant': 'float32'\n}\ntrain_csv = pd.read_csv('../input/champs-scalar-coupling/train.csv', index_col='id', dtype=train_dtypes)\ntrain_csv['molecule_index'] = train_csv.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\ntrain_csv = train_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type', 'scalar_coupling_constant']]\ntrain_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Shape: ', train_csv.shape)\nprint('Total: ', train_csv.memory_usage().sum())\ntrain_csv.memory_usage()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission_csv = pd.read_csv('../input/champs-scalar-coupling/sample_submission.csv', index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv = pd.read_csv('../input/champs-scalar-coupling/test.csv', index_col='id', dtype=train_dtypes)\ntest_csv['molecule_index'] = test_csv['molecule_name'].str.replace('dsgdb9nsd_', '').astype('int32')\ntest_csv = test_csv[['molecule_index', 'atom_index_0', 'atom_index_1', 'type']]\ntest_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures_dtypes = {\n    'molecule_name': 'category',\n    'atom_index': 'int8',\n    'atom': 'category',\n    'x': 'float32',\n    'y': 'float32',\n    'z': 'float32'\n}\nstructures_csv = pd.read_csv('../input/champs-scalar-coupling/structures.csv', dtype=structures_dtypes)\nstructures_csv['molecule_index'] = structures_csv.molecule_name.str.replace('dsgdb9nsd_', '').astype('int32')\nstructures_csv = structures_csv[['molecule_index', 'atom_index', 'atom', 'x', 'y', 'z']]\nstructures_csv['atom'] = structures_csv['atom'].replace(ATOMIC_NUMBERS).astype('int8')\nstructures_csv.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Shape: ', structures_csv.shape)\nprint('Total: ', structures_csv.memory_usage().sum())\nstructures_csv.memory_usage()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build Distance Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_type_dataframes(base, structures, coupling_type):\n    base = base[base['type'] == coupling_type].drop('type', axis=1).copy()\n    base = base.reset_index()\n    base['id'] = base['id'].astype('int32')\n    structures = structures[structures['molecule_index'].isin(base['molecule_index'])]\n    return base, structures","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_coordinates(base, structures, index):\n    df = pd.merge(base, structures, how='inner',\n                  left_on=['molecule_index', f'atom_index_{index}'],\n                  right_on=['molecule_index', 'atom_index']).drop(['atom_index'], axis=1)\n    df = df.rename(columns={\n        'atom': f'atom_{index}',\n        'x': f'x_{index}',\n        'y': f'y_{index}',\n        'z': f'z_{index}'\n    })\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_atoms(base, atoms):\n    df = pd.merge(base, atoms, how='inner',\n                  on=['molecule_index', 'atom_index_0', 'atom_index_1'])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def merge_all_atoms(base, structures):\n    df = pd.merge(base, structures, how='left',\n                  left_on=['molecule_index'],\n                  right_on=['molecule_index'])\n    df = df[(df.atom_index_0 != df.atom_index) & (df.atom_index_1 != df.atom_index)]\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_center(df):\n    df['x_c'] = ((df['x_1'] + df['x_0']) * np.float32(0.5))\n    df['y_c'] = ((df['y_1'] + df['y_0']) * np.float32(0.5))\n    df['z_c'] = ((df['z_1'] + df['z_0']) * np.float32(0.5))\n\ndef add_distance_to_center(df):\n    df['d_c'] = ((\n        (df['x_c'] - df['x'])**np.float32(2) +\n        (df['y_c'] - df['y'])**np.float32(2) + \n        (df['z_c'] - df['z'])**np.float32(2)\n    )**np.float32(0.5))\n\ndef add_distance_between(df, suffix1, suffix2):\n    df[f'd_{suffix1}_{suffix2}'] = ((\n        (df[f'x_{suffix1}'] - df[f'x_{suffix2}'])**np.float32(2) +\n        (df[f'y_{suffix1}'] - df[f'y_{suffix2}'])**np.float32(2) + \n        (df[f'z_{suffix1}'] - df[f'z_{suffix2}'])**np.float32(2)\n    )**np.float32(0.5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_distances(df):\n    n_atoms = 1 + max([int(c.split('_')[1]) for c in df.columns if c.startswith('x_')])\n    \n    for i in range(1, n_atoms):\n        for vi in range(min(4, i)):\n            add_distance_between(df, i, vi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_n_atoms(base, structures):\n    dfs = structures['molecule_index'].value_counts().rename('n_atoms').to_frame()\n    return pd.merge(base, dfs, left_on='molecule_index', right_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_couple_dataframe(some_csv, structures_csv, coupling_type, n_atoms=10):\n    base, structures = build_type_dataframes(some_csv, structures_csv, coupling_type)\n    base = add_coordinates(base, structures, 0)\n    base = add_coordinates(base, structures, 1)\n    \n    base = base.drop(['atom_0', 'atom_1'], axis=1)\n    atoms = base.drop('id', axis=1).copy()\n    if 'scalar_coupling_constant' in some_csv:\n        atoms = atoms.drop(['scalar_coupling_constant'], axis=1)\n        \n    add_center(atoms)\n    atoms = atoms.drop(['x_0', 'y_0', 'z_0', 'x_1', 'y_1', 'z_1'], axis=1)\n\n    atoms = merge_all_atoms(atoms, structures)\n    \n    add_distance_to_center(atoms)\n    \n    atoms = atoms.drop(['x_c', 'y_c', 'z_c', 'atom_index'], axis=1)\n    atoms.sort_values(['molecule_index', 'atom_index_0', 'atom_index_1', 'd_c'], inplace=True)\n    atom_groups = atoms.groupby(['molecule_index', 'atom_index_0', 'atom_index_1'])\n    atoms['num'] = atom_groups.cumcount() + 2\n    atoms = atoms.drop(['d_c'], axis=1)\n    atoms = atoms[atoms['num'] < n_atoms]\n\n    atoms = atoms.set_index(['molecule_index', 'atom_index_0', 'atom_index_1', 'num']).unstack()\n    atoms.columns = [f'{col[0]}_{col[1]}' for col in atoms.columns]\n    atoms = atoms.reset_index()\n    \n    # downcast back to int8\n    for col in atoms.columns:\n        if col.startswith('atom_'):\n            atoms[col] = atoms[col].fillna(0).astype('int8')\n            \n    atoms['molecule_index'] = atoms['molecule_index'].astype('int32')\n    \n    full = add_atoms(base, atoms)\n    add_distances(full)\n    \n    full.sort_values('id', inplace=True)\n    \n    return full","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def take_n_atoms(df, n_atoms, four_start=4):\n    labels = []\n    for i in range(2, n_atoms):\n        label = f'atom_{i}'\n        labels.append(label)\n\n    for i in range(n_atoms):\n        num = min(i, 4) if i < four_start else 4\n        for j in range(num):\n            labels.append(f'd_{i}_{j}')\n    if 'scalar_coupling_constant' in df:\n        labels.append('scalar_coupling_constant')\n    return df[labels]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Check XGB with the smallest type"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_csv_ = test_csv.copy()\ntest_csv_[\"scalar_coupling_constant\"] = \"unknown\"\nconcated = pd.concat([train_csv, test_csv_])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfull = build_couple_dataframe(concated, structures_csv, '1JHC', n_atoms=15)\nprint(full.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We don't calculate distances for `d_0_x`, `d_1_1`, `d_2_2`, `d_2_3`, `d_3_3` because we already have them in later atoms(`d_0_1` == `d_1_0`) or they are equal to zeros(e.g. `d_1_1`, `d_2_2`)."},{"metadata":{"trusted":true},"cell_type":"code","source":"full","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"full.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For experiments, full dataset can be built with higher number of atoms, and for building a training/validation sets we can trim them:"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = take_n_atoms(full, 7)\n# LightGBM performs better with 0-s then with NaN-s\ndf = df.fillna(0)\ndf.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_data = df.drop(['scalar_coupling_constant'], axis=1).values.astype('float32')\n# y_data = df['scalar_coupling_constant'].values.astype('float32')\ndf[\"std\"] = df.drop(['scalar_coupling_constant'], axis=1).values.std(axis=1)\ndf[\"std\"] = df[\"std\"].apply(lambda x: round(x, 1))\nX_data = df.sample(frac=1, random_state=43).drop(['scalar_coupling_constant'], axis=1).values\n\ny_data = df['scalar_coupling_constant'].values\n\nX_train, X_val, y_train, y_val = train_test_split(X_data, y_data, test_size=0.2, random_state=128)\nX_train.shape, X_val.shape, y_train.shape, y_val.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**First 4 item in train have std < 1. They are useless**"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = df.iloc[4:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom sklearn.manifold import TSNE\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objs as go\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\n\nX_embedded = TSNE(n_components=2, perplexity=25, random_state=50).fit_transform(X_data[:100000])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot by scalar_coupling_constant"},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_data = pd.DataFrame(data={\"x_axis\": X_embedded[:,0], \"y_axis\": X_embedded[:,1],\n                              \"scalar_coupling_constant\": y_data[:X_embedded.shape[0]]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_data[\"scalar_coupling_constant\"] = (tsne_data[\"scalar_coupling_constant\"]/10).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_2d(df, x, y):\n    plt.figure(figsize=(16,10))\n    sns.scatterplot(\n        x=x, y=y,\n        hue='scalar_coupling_constant',\n        palette=sns.color_palette('bright', tsne_data[\"scalar_coupling_constant\"].nunique()),\n        data=df,\n        legend='full',\n        alpha=0.9\n    )\n    plt.show()\n    \n\ndef plot_3d(df, x, y, z):\n    trace1 = go.Scatter3d(x=df[x].values, y=df[y].values, z=df[z].values,\n        mode='markers',\n        marker=dict(\n            color=df['scalar_coupling_constant'].values,\n            colorscale = \"Jet\",\n            opacity=0.,\n            size=2\n        )\n    )\n\n    figure_data = [trace1]\n    layout = go.Layout(\n        scene = dict(\n            xaxis = dict(title=x),\n            yaxis = dict(title=y),\n            zaxis = dict(title=z),\n        ),\n        margin=dict(\n            l=0,\n            r=0,\n            b=0,\n            t=0\n        ),\n        showlegend=True\n    )\n\n    fig = go.Figure(data=figure_data, layout=layout)\n    py.iplot(fig, filename='3d_scatter')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_2d(tsne_data, x=\"x_axis\", y=\"y_axis\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Plot by std"},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_data = pd.DataFrame(data={\"x_axis\": X_embedded[:,0], \"y_axis\": X_embedded[:,1],\n                              \"std\": df[\"std\"].values[:X_embedded.shape[0]]})\n\ndef plot_2d(df, x, y):\n    plt.figure(figsize=(16,10))\n    sns.scatterplot(\n        x=x, y=y,\n        hue='std',\n        palette=sns.color_palette('bright', tsne_data[\"std\"].nunique()),\n        data=df,\n        legend='full',\n        alpha=0.9\n    )\n    plt.show()\n    \n\ndef plot_3d(df, x, y, z):\n    trace1 = go.Scatter3d(x=df[x].values, y=df[y].values, z=df[z].values,\n        mode='markers',\n        marker=dict(\n            color=df['std'].values,\n            colorscale = \"Jet\",\n            opacity=0.,\n            size=2\n        )\n    )\n\n    figure_data = [trace1]\n    layout = go.Layout(\n        scene = dict(\n            xaxis = dict(title=x),\n            yaxis = dict(title=y),\n            zaxis = dict(title=z),\n        ),\n        margin=dict(\n            l=0,\n            r=0,\n            b=0,\n            t=0\n        ),\n        showlegend=True\n    )\n\n    fig = go.Figure(data=figure_data, layout=layout)\n    py.iplot(fig, filename='3d_scatter')\n    \nplot_2d(tsne_data, x=\"x_axis\", y=\"y_axis\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}