{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Auto Champs\n\nIn this notebook I aim to demonstrate automation on feature generation and feature selection. For that purpose I used featuretools to aggregate the data by `molecule_name` and `atom_index_0`/`atom_index_1` and automatically generated possible statistical features with a depth of 2.\n\nThen having the automatically generated features, I used borutaPy to select the best features for the model. And finally I train a lightgbm model using those features.\n\nThis notebook can be seen as an automated version of this work: https://www.kaggle.com/artgor/brute-force-feature-engineering\n\nAdditional example for featuretools: https://www.kaggle.com/willkoehrsen/automated-feature-engineering-tutorial\n\nAdditional example for borutaPy: https://www.kaggle.com/rsmits/feature-selection-with-boruta"},{"metadata":{"trusted":true},"cell_type":"code","source":"%ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_dir = '/kaggle/input/'\ndownload_dir = './'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"is_sample = False # if True, run in test mode\nboosting_rounds = 15000 # lightgbm training epochs\nboruta_max_iter = 50 # max iteration number for boruta\nnum_boruta_rows = 5000 # use a small subsample to quickly fit with boruta feature selector","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport featuretools as ft\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nimport gc\nimport lightgbm\nfrom sklearn.ensemble import RandomForestRegressor\nfrom boruta import BorutaPy\nfrom dask.distributed import LocalCluster\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# calculate competition metric\ndef competition_metric(df, preds, verbose=0):\n    # log of mean absolute error, calculated for each scalar coupling type.\n    df_copy = df.copy()\n    df_copy[\"prediction\"] = preds\n    maes = []\n    for t in df_copy.type.unique():\n        y_true = df_copy[df.type == t].scalar_coupling_constant.values\n        y_pred = df_copy[df.type == t].prediction.values\n        mae = np.log(metrics.mean_absolute_error(y_true, y_pred))\n        if verbose == 1:\n            print(f\"{t} log(MAE): {mae}\")\n        maes.append(mae)\n    del df_copy\n    gc.collect()\n    return np.mean(maes)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prepare the dataset\n\nI will read the datasets and craete basic features like the distance measurements and then concatenate train and test."},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(f\"{dataset_dir}train.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(f\"{dataset_dir}test.csv\")\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat = pd.concat([train, test])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures = pd.read_csv(f\"{dataset_dir}structures.csv\")\nstructures.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# map structures dataframe into concat\ndef map_atom_info(df, atom_idx):\n    df = pd.merge(df, structures, how = 'left',\n                  left_on  = ['molecule_name', f'atom_index_{atom_idx}'],\n                  right_on = ['molecule_name',  'atom_index'])\n    \n    df = df.drop('atom_index', axis=1)\n    df = df.rename(columns={'atom': f'atom_{atom_idx}',\n                            'x': f'x_{atom_idx}',\n                            'y': f'y_{atom_idx}',\n                            'z': f'z_{atom_idx}'})\n    return df\n\nconcat = map_atom_info(concat, 0)\nconcat = map_atom_info(concat, 1)\n\nconcat.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create basic features like distance\ndef particle_distance(df):\n    dist = ( (df[\"x_1\"] - df[\"x_0\"])**2 + (df[\"y_1\"] - df[\"y_0\"])**2 + (df[\"z_1\"] - df[\"z_0\"])**2 )**0.5\n    return dist\n\nconcat[\"distance\"] = particle_distance(concat)\n\n# create distance values for each axis\ndef particle_distance_x(df):\n    dist = ( (df[\"x_1\"] - df[\"x_0\"])**2 )**0.5\n    return dist\n\ndef particle_distance_y(df):\n    dist = ( (df[\"y_1\"] - df[\"y_0\"])**2 )**0.5\n    return dist\n\ndef particle_distance_z(df):\n    dist = ( (df[\"z_1\"] - df[\"z_0\"])**2 )**0.5\n    return dist\n\nconcat[\"distance_x\"] = particle_distance_x(concat)\nconcat[\"distance_y\"] = particle_distance_y(concat)\nconcat[\"distance_z\"] = particle_distance_z(concat)\n\nconcat.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if is_sample:\n    print(\"\\n!!! WARNING SAMPLE MODE ACTIVE !!!\\n\")\n    concat = concat[:1000]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Aggregation\n\nIn order to create stacked features with a depth more than 1, we need to define [relational entities](https://docs.featuretools.com/loading_data/using_entitysets.html). Since I want to aggregate by `molecule_name` + `atom_index_0` and `molecule_name` + `atom_index_1`, I will concat those columns and create the ids below."},{"metadata":{"trusted":true},"cell_type":"code","source":"le = preprocessing.LabelEncoder()\nmol_atom_0 = concat.molecule_name.astype(str) + '_' + concat.atom_index_0.astype(str)\nconcat['molecule_atom_0_id'] = le.fit_transform(mol_atom_0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = preprocessing.LabelEncoder()\nmol_atom_1 = concat.molecule_name.astype(str) + '_' + concat.atom_index_1.astype(str)\nconcat['molecule_atom_1_id'] = le.fit_transform(mol_atom_1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create the Entity Set\n\nIn order to use featuretools, I need to create an entity set and define the relations. I could do this by splitting the concat dataframe into 3 dataframes with two of them are for the concatenated molecule name and atom index number features. However there is an easier way to do it by [entitiy normalization](https://docs.featuretools.com/loading_data/using_entitysets.html#creating-entity-from-existing-table)."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Create the entity set for featuretools\nes = ft.EntitySet(id='concat')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Add entites to entity set\nes = es.entity_from_dataframe(\n    entity_id='concat', dataframe=concat.drop(['scalar_coupling_constant'], axis=1), index='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = es.normalize_entity(\n    base_entity_id='concat',\n    new_entity_id='molecule_atom_0',\n    index='molecule_atom_0_id',\n    additional_variables=['atom_0', 'x_0', 'y_0', 'z_0'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = es.normalize_entity(\n    base_entity_id='concat',\n    new_entity_id='molecule_atom_1',\n    index='molecule_atom_1_id',\n    additional_variables=['atom_1', 'x_1', 'y_1', 'z_1'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Generate Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"# It is faster when using n_jobs > 1, however kaggle kernels die if I define multiple jobs, so I comment out those lines below.\n#cluster = LocalCluster()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Perform an automated Deep Feature Synthesis with a depth of 2\n#features0, feature_names0 = ft.dfs(entityset=es, target_entity='molecule_atom_0', max_depth=2, dask_kwargs={'cluster': cluster}, n_jobs=2)\nfeatures0, feature_names0 = ft.dfs(entityset=es, target_entity='molecule_atom_0', max_depth=2)\nprint(features0.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Perform an automated Deep Feature Synthesis with a depth of 2\n#features1, feature_names1 = ft.dfs(entityset=es, target_entity='molecule_atom_1', max_depth=2, dask_kwargs={'cluster': cluster}, n_jobs=2)\nfeatures1, feature_names1 = ft.dfs(entityset=es, target_entity='molecule_atom_1', max_depth=2)\nprint(features1.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_names0","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_names1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# add column suffixes\ndef col_suffix_handler(df, suffix):\n    col_dict = {col:\"{}{}\".format(col, suffix) for col in df.columns.values}\n    df.rename(columns=col_dict, inplace=True)\n    return df\n\n# I will need unqiue feature names after feature selection with boruta\nfeatures0 = col_suffix_handler(features0, '__molecule_atom_0')\nfeatures1 = col_suffix_handler(features1, '__molecule_atom_1')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Select Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"# reduce memory\ndef reduce_memory(df):\n    num_converted_cols = 0\n    for col in df.columns.values:\n        if df[col].dtype == \"float64\":\n            num_converted_cols += 1\n            df[col] = df[col].astype(\"float32\")\n        elif df[col].dtype == \"int64\":\n            num_converted_cols += 1\n            df[col] = df[col].astype(\"int32\")\n    print(\"{} cols converted.\".format(num_converted_cols))\n    return df\n\nconcat = reduce_memory(concat)\nfeatures0 = reduce_memory(features0)\nfeatures1 = reduce_memory(features1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# handle NaN values\ndef nan_handler(df):\n    for col in df.columns.values:\n        if np.any(df[col].isnull()):\n            print(col)\n            if df[col].dtype == 'O':\n                df[col] = df[col].fillna('NO_VALUE')\n            else:\n                df[col] = df[col].fillna(-999)\n    return df\n                \nfeatures0 = nan_handler(features0)\nfeatures1 = nan_handler(features1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# handle inf/-inf values\ndef inf_handler(df):\n    for col in df.columns.values:\n        if np.any(df[col]==np.inf) or any(df[col]==-np.inf):\n            print(col)\n            if df[col].dtype == 'O':\n                df[df[col]==np.inf] = 'NO_VALUE'\n                df[df[col]==-np.inf] = 'NO_VALUE'\n            else:\n                df[df[col]==np.inf] = 999\n                df[df[col]==-np.inf] = 999\n    return df\n                \nfeatures0 = inf_handler(features0)\nfeatures1 = inf_handler(features1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# list unnecessary columns\ncols_to_remove = [\n    'id',\n    'scalar_coupling_constant'\n]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%%time\n\n# feature selection using boruta\n\n# merge features with concat df\nconcat_features_ = concat.iloc[:num_boruta_rows].merge(\n    features0, left_on=['molecule_atom_0_id'], right_index=True, how='left')\n\nconcat_features_ = concat_features_.iloc[:num_boruta_rows].merge(\n    features1, left_on=['molecule_atom_1_id'], right_index=True, how='left')\n\n# label encode object type (categorical) columns\nfor col in concat_features_.columns.values:\n    if concat_features_[col].dtype == 'O':\n        le = preprocessing.LabelEncoder()\n        concat_features_[col] = le.fit_transform(concat_features_[col])\n\nforest = RandomForestRegressor(n_jobs=-1)\n\nfeat_selector = BorutaPy(\n    forest, n_estimators='auto', verbose=2, random_state=42, max_iter=boruta_max_iter, perc=90)\n\nX = concat_features_.drop(cols_to_remove, axis=1).iloc[:num_boruta_rows, :].values\ny = concat_features_[[\"scalar_coupling_constant\"]].values[:num_boruta_rows, 0]\n\nfeat_selector.fit(X, y)\n\nfeatures = concat_features_.drop(cols_to_remove, axis=1).columns.values.tolist()\n\ndel X, y, concat_features_\ngc.collect()\n\n# list selected boruta features\nselected_features = []\nindexes = np.where(feat_selector.support_ == True)\nfor x in np.nditer(indexes):\n    selected_features.append(features[x])\n\nprint(len(selected_features))\nprint(selected_features)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Lightgbm Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# merge features0 and features1 with concat df (using only selected features)\n\nselected_features0_ = list(set(selected_features) - set(concat.columns.values.tolist()))\nselected_features0_ = [f for f in selected_features0_ if '__molecule_atom_0' in f]\n\nselected_features1_ = list(set(selected_features) - set(concat.columns.values.tolist()))\nselected_features1_ = [f for f in selected_features1_ if '__molecule_atom_1' in f]\n\nconcat_features = concat.merge(\n    features0[selected_features0_], \n    left_on=['molecule_atom_0_id'], \n    right_index=True, \n    how='left'\n)\n\nconcat_features = concat_features.merge(\n    features1[selected_features1_], \n    left_on=['molecule_atom_1_id'], \n    right_index=True,\n    how='left'\n)\n\nprint(concat_features.shape)\nconcat_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"concat_features.dtypes.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# label encode object type columns\nfor col in concat_features.columns.values:\n    if concat_features[col].dtype == 'O':\n        le = preprocessing.LabelEncoder()\n        concat_features[col] = le.fit_transform(concat_features[col])\n        \nconcat_features.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len_train = len(train)\ndel train, test, concat, features0, features1\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = concat_features[:len_train]\ntest = concat_features[len_train:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del concat_features\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# use selected boruta features and train a lightgbm model\n\ntrain_index, valid_index = train_test_split(np.arange(len(train)),random_state=42, test_size=0.1)\n\nX_train = train[selected_features].values[train_index]\ny_train = train[['scalar_coupling_constant']].values[:, 0][train_index]\n\nvalid_df = train.iloc[valid_index]\n\ndel train\ngc.collect()\n\nX_valid = valid_df[selected_features].values\ny_valid = valid_df[['scalar_coupling_constant']].values[:, 0]\n\nparams = {'boosting': 'gbdt', 'colsample_bytree': 1, \n              'learning_rate': 0.1, 'max_depth': 40, 'metric': 'mae',\n              'min_child_samples': 50, 'num_leaves': 500, \n              'objective': 'regression', 'reg_alpha': 0.8, \n              'reg_lambda': 0.8, 'subsample': 0.5 }\n\nlgtrain = lightgbm.Dataset(X_train, label=y_train)\nlgval = lightgbm.Dataset(X_valid, label=y_valid)\n\nmodel_lgb = lightgbm.train(\n    params, lgtrain, boosting_rounds, valid_sets=[lgtrain, lgval], \n    early_stopping_rounds=1000, verbose_eval=500)\n\n# evaluate using validation set\nevals = model_lgb.predict(X_valid)\nlmae = competition_metric(valid_df, evals, verbose=1)\nprint(\"Log of MAE = {}\".format(lmae))\n\ndel valid_df, X_train, y_train, X_valid, y_valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Predict"},{"metadata":{"trusted":true},"cell_type":"code","source":"# predict for test set\nX_test = test[selected_features].values\npreds = model_lgb.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# save predictions\ntest[\"scalar_coupling_constant\"] = preds\ntest[[\"id\", \"scalar_coupling_constant\"]].to_csv(f\"{download_dir}preds.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}