{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Simple EDA and LightGBM model\n* 5-fold cross validation\n* easy feature engineering and EDA"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nimport seaborn as sns\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold,StratifiedKFold\nfrom sklearn.preprocessing import LabelEncoder\nimport matplotlib.pyplot as plt\nfrom sklearn import metrics\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train / Test Set"},{"metadata":{},"cell_type":"markdown","source":"the training set, where the first column (molecule_name) is the name of the molecule where the coupling constant originates (the corresponding XYZ file is located at ./structures/.xyz), the second (atom_index_0) and third column (atom_index_1) is the atom indices of the atom-pair creating the coupling and the fourth column (scalar_coupling_constant) is the scalar coupling constant that we want to be able to predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print (\"Total Train Set : %d\" %len(train))\nprint ('Total Test Set : %d' %len(test))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Target : scalar_coupling_constant\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['scalar_coupling_constant'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.distplot(train['scalar_coupling_constant'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot type / target correlation\nLooks each type have some correlate to target"},{"metadata":{"trusted":true},"cell_type":"code","source":"typelist = list(train['type'].value_counts().index)\ntypelist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(26, 24))\nfor i, col in enumerate(typelist):\n    plt.subplot(4,2, i + 1)\n    sns.distplot(train[train['type']==col]['scalar_coupling_constant'],color ='orange')\n    plt.title(col)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Structure X/Y/Z"},{"metadata":{"trusted":true},"cell_type":"code","source":"structures = pd.read_csv('../input/structures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Merge the train set and structure set"},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/inversion/atomic-distance-benchmark/output\ndef map_atom_info(df, atom_idx):\n    df = pd.merge(df, structures, how = 'left',\n                  left_on  = ['molecule_name', f'atom_index_{atom_idx}'],\n                  right_on = ['molecule_name',  'atom_index'])\n    \n    df = df.drop('atom_index', axis=1)\n    df = df.rename(columns={'atom': f'atom_{atom_idx}',\n                            'x': f'x_{atom_idx}',\n                            'y': f'y_{atom_idx}',\n                            'z': f'z_{atom_idx}'})\n    return df\n\ntrain = map_atom_info(train, 0)\ntrain = map_atom_info(train, 1)\n\ntest = map_atom_info(test, 0)\ntest = map_atom_info(test, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## FE - Distance of atom"},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/inversion/atomic-distance-benchmark/output\n\ntrain['dist'] = ((train['x_1'] - train['x_0'])**2 +\n             (train['y_1'] - train['y_0'])**2 +\n             (train['z_1'] - train['z_0'])**2 ) ** 0.5\n\ntest['dist'] = ((test['x_1'] - test['x_0'])**2 +\n             (test['y_1'] - test['y_0'])**2 +\n             (test['z_1'] - test['z_0'])**2 ) ** 0.5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Label Encoding"},{"metadata":{"trusted":true},"cell_type":"code","source":"molecules = train.pop('molecule_name')\ntest = test.drop('molecule_name', axis=1)\nid_train = train.pop('id')\nid_test = test.pop('id')\n\ny = train.pop('scalar_coupling_constant')\n\n# Label Encoding\nfor f in ['type', 'atom_0', 'atom_1']:\n    lbl = LabelEncoder()\n    lbl.fit(list(train[f].values) + list(test[f].values))\n    train[f] = lbl.transform(list(train[f].values))\n    test[f] = lbl.transform(list(test[f].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Evaluate matric\n## https://www.kaggle.com/abhishek/competition-metric\ndef metric(df, preds):\n    df[\"prediction\"] = preds\n    maes = []\n    for t in df.type.unique():\n        y_true = df[df.type==t].scalar_coupling_constant.values\n        y_pred = df[df.type==t].prediction.values\n        mae = np.log(metrics.mean_absolute_error(y_true, y_pred))\n        maes.append(mae)\n    return np.mean(maes)\n#df for evaluate\neval_df = pd.DataFrame({\"type\":train[\"type\"]})\neval_df[\"scalar_coupling_constant\"] = y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_splits = 5 # Number of K-fold Splits\n\nsplits = list(GroupKFold(n_splits=n_splits).split(train, y, groups=molecules))\nsplits[:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"params = {\"learning_rate\" : 0.1,\n          \"depth\": 9,\n          'metric':'MAE',\n          'min_samples_leaf': 3,\n          \"loss_function\": \"MAE\"}","execution_count":1,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof = np.zeros(len(train))\npredictions = np.zeros(len(test))\nfeature_importance_df = pd.DataFrame()\nfeatures = [c for c in train.columns if c not in ['id']]\n\nfor i, (train_idx, valid_idx) in enumerate(splits):  \n    print(f'Fold {i + 1}')\n    x_train = np.array(train)\n    y_train = np.array(y)\n    trn_data = lgb.Dataset(x_train[train_idx.astype(int)], label=y_train[train_idx.astype(int)])\n    val_data = lgb.Dataset(x_train[valid_idx.astype(int)], label=y_train[valid_idx.astype(int)])\n    \n    num_round = 10000\n    clf = lgb.train(params, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=500, early_stopping_rounds = 200)\n    oof[valid_idx] = clf.predict(x_train[valid_idx], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = i + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    #predictions[fake_data.index] += clf.predict(fake_data, num_iteration=clf.best_iteration) / n_splits\n    predictions += clf.predict(test, num_iteration=clf.best_iteration) / n_splits\n\n#print(\"CV score: {:<8.5f}\".format(np.log(metrics.mean_absolute_error(train, oof))))","execution_count":2,"outputs":[{"output_type":"error","ename":"NameError","evalue":"name 'np' is not defined","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)","\u001b[0;32m<ipython-input-2-1d2a9525e296>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0moof\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mpredictions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mfeature_importance_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mDataFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mc\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mc\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mNameError\u001b[0m: name 'np' is not defined"]}]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"CV score: {:<8.5f}\".format(metric(eval_df, oof)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot feature important"},{"metadata":{"trusted":true},"cell_type":"code","source":"cols = (feature_importance_df[[\"feature\", \"importance\"]]\n        .groupby(\"feature\")\n        .mean()\n        .sort_values(by=\"importance\", ascending=False)[:1000].index)\nbest_features = feature_importance_df.loc[feature_importance_df.feature.isin(cols)]\n\nplt.figure(figsize=(14,5))\nsns.barplot(x=\"importance\", y=\"feature\", data=best_features.sort_values(by=\"importance\",ascending=False))\nplt.title('LightGBM Features (averaged over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importances.png')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\n\nbenchmark = sample_submission.copy()\nbenchmark['scalar_coupling_constant'] = predictions\nbenchmark.to_csv('LGBM_submission.csv',index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"benchmark.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Future work\n* EDA on Additional data\n* More FE\n* NN model build"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}