{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\ndev_df = pd.read_csv(\"../input/champs-scalar-coupling/train.csv\")\ntest_df = pd.read_csv(\"../input/champs-scalar-coupling/test.csv\")\n\ndev_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dev_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dev_df[\"median_error\"] = np.abs(dev_df[\"scalar_coupling_constant\"] - dev_df.groupby(\"type\")[\"scalar_coupling_constant\"].transform(\"median\"))\n\nstats = dev_df.groupby(\"type\").agg({\"id\": \"count\", \"median_error\": \"mean\"}).reset_index()\nstats[\"effect\"] = stats[\"id\"]*stats[\"median_error\"]\nstats[\"effect\"] /= stats[\"effect\"].min()\nstats[\"sw\"] = 1/stats[\"effect\"]\n\ndev_df = dev_df.merge(stats[[\"type\", \"sw\"]], on=\"type\", how=\"left\")\n\ndel stats","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder\n\nle_type = LabelEncoder()\ndev_df[\"le_type\"] = le_type.fit_transform(dev_df[\"type\"].values)\ntest_df[\"le_type\"] = le_type.transform(test_df[\"type\"].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures = pd.read_csv(\"../input/champs-scalar-coupling/structures.csv\")\nstructures.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_loc(df):\n    df = df.merge(structures.rename(columns={\"x\": \"x0\", \"y\": \"y0\", \"z\":\"z0\", \"atom\": \"atom0\", \"atom_index\": \"atom_index_0\"}), \n                          on=[\"molecule_name\", \"atom_index_0\"], how=\"left\")\n    df = df.merge(structures.rename(columns={\"x\": \"x1\", \"y\": \"y1\", \"z\":\"z1\", \"atom\": \"atom1\", \"atom_index\": \"atom_index_1\"}), \n                          on=[\"molecule_name\", \"atom_index_1\"], how=\"left\")\n    return df\n\ndev_df = get_loc(dev_df)\ntest_df = get_loc(test_df)\ndev_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mol_df = structures.groupby(\"molecule_name\")[\"atom\"].count().reset_index().rename(columns={\"atom\": \"atom_count\"})\n\ndef get_features(df):\n    df = df.merge(mol_df, on=\"molecule_name\", how=\"left\")\n    df[\"dist\"] = 1.0/np.linalg.norm(df[[\"x0\", \"y0\", \"z0\"]].values - df[[\"x1\", \"y1\", \"z1\"]].values, axis=1)\n    return df\n\ndev_df = get_features(dev_df)\ntest_df = get_features(test_df)\ndev_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures[\"mol_features\"] = structures[[\"x\", \"y\", \"z\"]].values.tolist()\nmol_df = structures.groupby(\"molecule_name\").agg({\"mol_features\": list, \"atom\": list}).reset_index()\n\ndev_df = dev_df.merge(mol_df, on=\"molecule_name\", how=\"left\")\ntest_df = test_df.merge(mol_df, on=\"molecule_name\", how=\"left\")\n\ndev_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in dev_df.columns:\n    if dev_df[col].dtype == np.int64:\n        dev_df[col] = dev_df[col].astype(np.int32)\n        \n    if dev_df[col].dtype == np.float64:\n        dev_df[col] = dev_df[col].astype(np.float32)\n        \nfor col in test_df.columns:\n    if test_df[col].dtype == np.int64:\n        test_df[col] = test_df[col].astype(np.int32)\n        \n    if test_df[col].dtype == np.float64:\n        test_df[col] = test_df[col].astype(np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from tqdm import tqdm, tqdm_notebook\nfrom sklearn.metrics.pairwise import euclidean_distances\n\ndef get_mol(df):\n    M = np.zeros((df.shape[0], 30, 5), dtype=np.float32)\n\n    index = 0\n\n    for properties, cx, mol_features, atom in tqdm_notebook(list(zip(df[[\"atom_index_0\", \"atom_index_1\"]].values, \n                                                          df[[\"x0\", \"y0\", \"z0\", \"x1\", \"y1\", \"z1\"]].values, \n                                                          df[\"mol_features\"].values, df[\"atom\"].values))):\n        arr = np.array(mol_features)\n        arr = np.delete(arr, np.array([properties[0], properties[1]]), 0)\n\n        atoms = np.delete(np.array(atom), np.array([properties[0], properties[1]]), 0)\n\n        v0 = arr - np.array([cx[0], cx[1], cx[2]])\n        v1 = arr - np.array([cx[3], cx[4], cx[5]])\n\n        dist0 = np.sqrt(np.power(v0, 2).sum(axis=1))\n        dist1 = np.sqrt(np.power(v1, 2).sum(axis=1))\n\n        M[index, :arr.shape[0], 0] = 1.0/dist0\n        M[index, :arr.shape[0], 1] = 1.0/dist1\n        M[index, :arr.shape[0], 2] = (atoms == \"H\")*1.0\n        M[index, :arr.shape[0], 3] = (atoms != \"H\")*1.0\n        M[index, :arr.shape[0], 4] = np.sum(v0*v1, axis=1)/(dist0*dist1)\n        index += 1\n        \n    return M\n\nM_train = get_mol(dev_df)\nM_test = get_mol(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from keras.layers import *\nfrom keras.models import Model\nfrom keras.optimizers import Nadam\n\nfeatures = [\"dist\", \"atom_count\"]\n\ndef nn_block(input_layer, size, dropout_rate, activation):\n    out_layer = Dense(size, activation=None)(input_layer)\n    out_layer = BatchNormalization()(out_layer)\n    out_layer = Activation(activation)(out_layer)\n    out_layer = Dropout(dropout_rate)(out_layer)\n    return out_layer\n\ndef cnn_block(input_layer, size, dropout_rate, activation):\n    out_layer = Conv1D(size, 1, activation=None)(input_layer)\n    out_layer = BatchNormalization()(out_layer)\n    out_layer = Activation(activation)(out_layer)\n    out_layer = Dropout(dropout_rate)(out_layer)\n    return out_layer\n\n\ndef get_model():\n    dense_input = Input(shape=(len(features) + 2,))\n\n    type_input = Input(shape=(1,))\n    type_emb = Flatten()(Embedding(dev_df[\"le_type\"].max() + 1, 5)(type_input))\n    \n    mol_input = Input(shape=(M_train.shape[1], M_train.shape[2]))\n    mol_layer = cnn_block(mol_input, 200, 0.0, \"relu\")\n    mol_layer = cnn_block(mol_layer, 100, 0.0, \"relu\")\n    \n    merged_input = BatchNormalization()(concatenate([dense_input, type_emb, \n                                                     GlobalMaxPooling1D()(mol_layer), GlobalAvgPool1D()(mol_layer)]))\n    \n    x1 = Dense(200, activation=\"relu\")(merged_input)\n    x2 = nn_block(x1, 20, 0.0, \"sigmoid\")\n    \n    mol_layer = concatenate([RepeatVector(30)(x2), mol_layer])\n    mol_layer = cnn_block(mol_layer, 200, 0.0, \"relu\")\n    mol_layer = cnn_block(mol_layer, 100, 0.0, \"relu\")\n    merged_input = BatchNormalization()(concatenate([dense_input, type_emb, \n                                                     GlobalMaxPooling1D()(mol_layer), GlobalAvgPool1D()(mol_layer)]))\n    hidden_layer = concatenate([Dense(600, activation=\"relu\")(merged_input), x1])\n    \n    hidden_layer = nn_block(hidden_layer, 400, 0.0, \"relu\")\n    hidden_layer = nn_block(hidden_layer, 200, 0.0, \"relu\")\n    hidden_layer = nn_block(hidden_layer, 100, 0.0, \"relu\")\n    \n    out = Dense(1, activation=\"linear\")(hidden_layer)\n    \n    model = Model(inputs=[dense_input, type_input, mol_input], outputs=out)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"import gc\n\nTARGET = \"scalar_coupling_constant\"\nkeep_list = [TARGET, \"id\", \"molecule_name\", \"le_type\", \"sw\", \"type\"] + features\ndev_df.drop([col for col in dev_df.columns if col not in keep_list], \n            axis=1, inplace=True)\ntest_df.drop([col for col in test_df.columns if col not in keep_list], \n             axis=1, inplace=True)\n\ndel mol_df, structures\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\n\nNUM_FOLDS = 4\nBATCH_SIZE = 2**10\nbest_model_path = \"best_model.h5\"\n\ndef to_emb(s):\n    return s.values.reshape(-1, 1)\n\ndef get_features(df):\n    return np.concatenate([df[features].values, np.log(df[features].values)], axis=1)\n\nkfold = GroupKFold(NUM_FOLDS)\noof_preds = np.zeros(dev_df.shape[0])\ntest_preds = np.zeros(test_df.shape[0])\n\nfor train_index, val_index in kfold.split(dev_df[TARGET].values, dev_df[TARGET].values, dev_df[\"molecule_name\"].values):\n    \n    train_df = dev_df[features + [\"sw\", \"le_type\", TARGET]].iloc[train_index]\n    val_df = dev_df[features + [\"sw\", \"le_type\", TARGET]].iloc[val_index]\n    \n    train_df[\"sw\"] *= train_df.shape[0]/train_df[\"sw\"].sum()\n    val_df[\"sw\"] *= val_df.shape[0]/val_df[\"sw\"].sum()\n    \n    model = get_model()\n    #early_stopping = EarlyStopping(monitor=\"val_loss\", patience=10, verbose=1)\n    model_checkpoint = ModelCheckpoint(best_model_path, save_best_only=True, save_weights_only=True, verbose=1)\n    \n    for epochs, lr in zip([3, 3, 3, 3, 6], [0.004, 0.004, 0.001, 0.001, 0.0004]):\n        model.compile(loss=\"mean_absolute_error\", optimizer=Nadam(lr=lr))\n        model.fit([get_features(train_df), to_emb(train_df[\"le_type\"]), M_train[train_index]], train_df[TARGET], \n                  sample_weight=train_df[\"sw\"].values,\n                  validation_data=([get_features(val_df), to_emb(val_df[\"le_type\"]), M_train[val_index]], val_df[TARGET], val_df[\"sw\"].values),\n                  verbose=1, epochs=epochs, batch_size=BATCH_SIZE, callbacks=[model_checkpoint])\n    \n    model.load_weights(best_model_path)\n    \n    oof_preds[val_index] = model.predict([get_features(val_df), to_emb(val_df[\"le_type\"]), M_train[val_index]], \n                                         verbose=1, batch_size=BATCH_SIZE).ravel()\n    \n    test_preds += model.predict([get_features(test_df), to_emb(test_df[\"le_type\"]), M_test], \n                                         verbose=1, batch_size=BATCH_SIZE).ravel()/NUM_FOLDS","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"del M_train, M_test\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"dev_df[\"oof_pred\"] = oof_preds\n\ndef competition_metric(df, pred_col):\n    df[\"error\"] = np.abs(df[TARGET] - df[pred_col])\n    return np.log(df.groupby(\"type\")[\"error\"].mean()).mean()\n\ncompetition_metric(dev_df, \"oof_pred\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_df[TARGET] = test_preds\ntest_df.to_csv(\"submission.csv\", columns=[\"id\", TARGET], index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0cac0b1e07394112aa7899426cc7e825":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_e2ec836adeb7430085d9121810df2504","IPY_MODEL_c6177c3959c14eb483de85a3ddd10804"],"layout":"IPY_MODEL_fa6467d886454e46a0db2c580464bf1c"}},"17f73a2498cd4dd78bbc2c543310129f":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"2c1607223870430c9774576596d555ae":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"90756f5476604ba0a2c27cd8faae4ee6":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"StyleView","description_width":""}},"b6c67b6c930249249472c3c941d15957":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c6177c3959c14eb483de85a3ddd10804":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_2c1607223870430c9774576596d555ae","placeholder":"​","style":"IPY_MODEL_90756f5476604ba0a2c27cd8faae4ee6","value":" 90% 4185306/4658147 [11:22<01:16, 6176.68it/s]"}},"e2ec836adeb7430085d9121810df2504":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.4.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.4.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.4.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_b6c67b6c930249249472c3c941d15957","max":4658147,"min":0,"orientation":"horizontal","style":"IPY_MODEL_17f73a2498cd4dd78bbc2c543310129f","value":4185306}},"fa6467d886454e46a0db2c580464bf1c":{"model_module":"@jupyter-widgets/base","model_module_version":"1.1.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.1.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.1.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}