{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is simple code that I tried to explore the data and build regression models."},{"metadata":{},"cell_type":"markdown","source":"Import libraries."},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import KFold, RandomizedSearchCV\nfrom sklearn.metrics import mean_absolute_error\nfrom sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor\nfrom sklearn.utils import resample\nimport xgboost as xgb\nimport matplotlib.pyplot as plt\nfrom IPython.display import clear_output\nimport time, datetime\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"PATH_TRAIN = '../input/train.csv'\nPATH_TEST =  '../input/test.csv'\nPATH_STR = '../input/structures.csv'\n\ntrain = pd.read_csv(PATH_TRAIN)\nLEN_TRAIN = len(train)\ntest = pd.read_csv(PATH_TEST)\nLEN_TEST = len(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Used mapping code from competition's benchmark kernel.(https://www.kaggle.com/inversion/atomic-distance-benchmark/) "},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# map structure information into train/test set.\nstructures = pd.read_csv(PATH_STR)\ndef map_atom_info(df, atom_idx):\n    df = pd.merge(df, structures, how = 'left',\n                  left_on  = ['molecule_name', f'atom_index_{atom_idx}'],\n                  right_on = ['molecule_name',  'atom_index'])\n    \n    df = df.drop('atom_index', axis=1)\n    df = df.rename(columns={'atom': f'atom_{atom_idx}',\n                            'x': f'x_{atom_idx}',\n                            'y': f'y_{atom_idx}',\n                            'z': f'z_{atom_idx}'})\n    return df\n\nstart = time.time()\n\ntrain = map_atom_info(train, 0)\ntrain = map_atom_info(train, 1)\n\ntest = map_atom_info(test, 0)\ntest = map_atom_info(test, 1)\n\nprint(str(datetime.timedelta(seconds=time.time()-start)))\nprint(train.columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Before we start, we can reduce memory-usage from dataframe."},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df\n\ntrain = reduce_mem_usage(train)\ntest = reduce_mem_usage(test)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We need more feature columns then i generated most outstanding and simple features.(distance information)  \n(thanks to sharing way to speed up calculation.(https://www.kaggle.com/seriousran/just-speed-up-calculate-distance-from-benchmark))"},{"metadata":{"trusted":true},"cell_type":"code","source":"# get new distance columns\nstart = time.time()\ntrain_c0 = train[['x_0', 'y_0', 'z_0']].values\ntrain_c1 = train[['x_1', 'y_1', 'z_1']].values\ntrain['dist'] = np.linalg.norm(train_c0 - train_c1, axis=1)\ntest_c0 = test[['x_0', 'y_0', 'z_0']].values\ntest_c1 = test[['x_1', 'y_1', 'z_1']].values\ntest['dist'] = np.linalg.norm(test_c0 - test_c1, axis=1)\n\ntrain['x_dist'] = (train['x_0'] - train['x_1']) ** 2\ntrain['y_dist'] = (train['y_0'] - train['y_1']) ** 2\ntrain['z_dist'] = (train['z_0'] - train['z_1']) ** 2\ntest['x_dist'] = (test['x_0'] - test['x_1']) ** 2\ntest['y_dist'] = (test['y_0'] - test['y_1']) ** 2\ntest['z_dist'] = (test['z_0'] - test['z_1']) ** 2\n\ntrain['xy_dist'] = np.linalg.norm(train[['x_0', 'y_0']].values - train[['x_1', 'y_1']].values, axis=1)\ntrain['xz_dist'] = np.linalg.norm(train[['x_0', 'z_0']].values - train[['x_1', 'z_1']].values, axis=1)\ntrain['yz_dist'] = np.linalg.norm(train[['y_0', 'z_0']].values - train[['y_1', 'z_1']].values, axis=1)\ntest['xy_dist'] = np.linalg.norm(test[['x_0', 'y_0']].values - test[['x_1', 'y_1']].values, axis=1)\ntest['xz_dist'] = np.linalg.norm(test[['x_0', 'z_0']].values - test[['x_1', 'z_1']].values, axis=1)\ntest['yz_dist'] = np.linalg.norm(test[['y_0', 'z_0']].values - test[['y_1', 'z_1']].values, axis=1)\nprint(str(datetime.timedelta(seconds=time.time()-start)))\nprint(train.columns)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"  "},{"metadata":{},"cell_type":"markdown","source":"Plotting some data distributions."},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(15, 5))\nfig.suptitle('Counts of types')\nax1 = fig.add_subplot(121)\nax1.set_ylim(0, 1600000)\nax1.bar(train['type'].value_counts().index, train['type'].value_counts(sort=False).values, color='k', width=0.5)\nax1.title.set_text('Train set')\n\nax2 = fig.add_subplot(122)\nax2.set_ylim(0, 1600000)\nax2.bar(test['type'].value_counts().index, test['type'].value_counts(sort=False).values, color='r', width=0.5)\nax2.title.set_text('Test set')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Maybe we can seperate regression models to every atom types."},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"fig = plt.figure(figsize=(15, 5))\nfig.suptitle('Counts of atom1')\n\nax1 = fig.add_subplot(121)\nax1.set_ylim(0, 4000000)\nax1.bar(train['atom_1'].value_counts().index, train['atom_1'].value_counts(sort=False).values, color='k', width=0.5)\nax1.title.set_text('Train set_atom1')\n\nax2 = fig.add_subplot(122)\nax2.set_ylim(0, 4000000)\nax2.bar(test['atom_1'].value_counts().index, test['atom_1'].value_counts(sort=False).values, color='r', width=0.5)\nax2.title.set_text('Test set_atom1')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.hist(column='scalar_coupling_constant', bins=20, color='k', grid=False)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"train.hist(column='scalar_coupling_constant', by='type', figsize=(10, 10), bins=20, color='k')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.hist(column='scalar_coupling_constant', by='atom_1', figsize=(7, 7), bins=20, color='k')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"resample(train, n_samples=10000).plot(kind='scatter', x='dist', y='scalar_coupling_constant', color='k', s=10, alpha=0.3)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"rand_idx = np.random.randint(0, LEN_TRAIN, 1000)\nfig = plt.figure(figsize=(15, 5))\nfig.suptitle('Cordinate values and scalar coupling')\nax1 = fig.add_subplot(121)\ntrain.iloc[rand_idx].plot(kind='scatter', x='x_0', y='scalar_coupling_constant', color='r', s=5, alpha=0.3, label='x_0', ax=ax1)\ntrain.iloc[rand_idx].plot(kind='scatter', x='y_0', y='scalar_coupling_constant', color='g', s=5, alpha=0.3, label='y_0', ax=ax1)\ntrain.iloc[rand_idx].plot(kind='scatter', x='z_0', y='scalar_coupling_constant', color='b', s=5, alpha=0.3, label='z_0', ax=ax1)\nax1.set_xlabel('cordinate 0')\nax2 = fig.add_subplot(122)\ntrain.iloc[rand_idx].plot(kind='scatter', x='x_1', y='scalar_coupling_constant', color='r', s=5, alpha=0.3, label='x_1', ax=ax2)\ntrain.iloc[rand_idx].plot(kind='scatter', x='y_1', y='scalar_coupling_constant', color='g', s=5, alpha=0.3, label='y_1', ax=ax2)\ntrain.iloc[rand_idx].plot(kind='scatter', x='z_1', y='scalar_coupling_constant', color='b', s=5, alpha=0.3, label='z_1', ax=ax2)\nax2.set_xlabel('cordinate 1')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"rand_idx = np.random.randint(0, LEN_TRAIN, 1000)\nfig = plt.figure(figsize=(15, 5))\nfig.suptitle('1-d, 2-d disntaces and scalar coupling')\nax1 = fig.add_subplot(121)\ntrain.iloc[rand_idx].plot(kind='scatter', x='x_dist', y='scalar_coupling_constant', color='r', s=5, alpha=0.3, label='x_dist', ax=ax1)\ntrain.iloc[rand_idx].plot(kind='scatter', x='y_dist', y='scalar_coupling_constant', color='g', s=5, alpha=0.3, label='y_dist', ax=ax1)\ntrain.iloc[rand_idx].plot(kind='scatter', x='z_dist', y='scalar_coupling_constant', color='b', s=5, alpha=0.3, label='z_dist', ax=ax1)\nax1.set_xlabel('1d_dist')\nax2 = fig.add_subplot(122)\ntrain.iloc[rand_idx].plot(kind='scatter', x='xy_dist', y='scalar_coupling_constant', color='r', s=5, alpha=0.3, label='xy_dist', ax=ax2)\ntrain.iloc[rand_idx].plot(kind='scatter', x='xz_dist', y='scalar_coupling_constant', color='g', s=5, alpha=0.3, label='xz_dist', ax=ax2)\ntrain.iloc[rand_idx].plot(kind='scatter', x='yz_dist', y='scalar_coupling_constant', color='b', s=5, alpha=0.3, label='yz_dist', ax=ax2)\nax2.set_xlabel('2d_dist')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Some of new features seem to be helpful a little bit, but we gotta prepare more features.  "},{"metadata":{},"cell_type":"markdown","source":"So we can generate various features from given information.  \n(thanks to sharing it.(https://www.kaggle.com/artgor/molecular-properties-eda-and-models))"},{"metadata":{"trusted":true},"cell_type":"code","source":"train['type_0'] = train['type'].apply(lambda x: int(x[0]))\ntest['type_0'] = test['type'].apply(lambda x: int(x[0]))\ndef create_more_features(df):\n    df['molecule_couples'] = df.groupby('molecule_name')['id'].transform('count')\n    df['molecule_dist_mean'] = df.groupby('molecule_name')['dist'].transform('mean')\n    df['molecule_dist_min'] = df.groupby('molecule_name')['dist'].transform('min')\n    df['molecule_dist_max'] = df.groupby('molecule_name')['dist'].transform('max')\n    df['atom_0_couples_count'] = df.groupby(['molecule_name', 'atom_index_0'])['id'].transform('count')\n    df['atom_1_couples_count'] = df.groupby(['molecule_name', 'atom_index_1'])['id'].transform('count')\n    df[f'molecule_atom_index_0_x_1_std'] = df.groupby(['molecule_name', 'atom_index_0'])['x_1'].transform('std')\n    df[f'molecule_atom_index_0_y_1_mean'] = df.groupby(['molecule_name', 'atom_index_0'])['y_1'].transform('mean')\n    df[f'molecule_atom_index_0_y_1_mean_diff'] = df[f'molecule_atom_index_0_y_1_mean'] - df['y_1']\n    df[f'molecule_atom_index_0_y_1_mean_div'] = df[f'molecule_atom_index_0_y_1_mean'] / df['y_1']\n    df[f'molecule_atom_index_0_y_1_max'] = df.groupby(['molecule_name', 'atom_index_0'])['y_1'].transform('max')\n    df[f'molecule_atom_index_0_y_1_max_diff'] = df[f'molecule_atom_index_0_y_1_max'] - df['y_1']\n    df[f'molecule_atom_index_0_y_1_std'] = df.groupby(['molecule_name', 'atom_index_0'])['y_1'].transform('std')\n    df[f'molecule_atom_index_0_z_1_std'] = df.groupby(['molecule_name', 'atom_index_0'])['z_1'].transform('std')\n    df[f'molecule_atom_index_0_dist_mean'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('mean')\n    df[f'molecule_atom_index_0_dist_mean_diff'] = df[f'molecule_atom_index_0_dist_mean'] - df['dist']\n    df[f'molecule_atom_index_0_dist_mean_div'] = df[f'molecule_atom_index_0_dist_mean'] / df['dist']\n    df[f'molecule_atom_index_0_dist_max'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('max')\n    df[f'molecule_atom_index_0_dist_max_diff'] = df[f'molecule_atom_index_0_dist_max'] - df['dist']\n    df[f'molecule_atom_index_0_dist_max_div'] = df[f'molecule_atom_index_0_dist_max'] / df['dist']\n    df[f'molecule_atom_index_0_dist_min'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('min')\n    df[f'molecule_atom_index_0_dist_min_diff'] = df[f'molecule_atom_index_0_dist_min'] - df['dist']\n    df[f'molecule_atom_index_0_dist_min_div'] = df[f'molecule_atom_index_0_dist_min'] / df['dist']\n    df[f'molecule_atom_index_0_dist_std'] = df.groupby(['molecule_name', 'atom_index_0'])['dist'].transform('std')\n    df[f'molecule_atom_index_0_dist_std_diff'] = df[f'molecule_atom_index_0_dist_std'] - df['dist']\n    df[f'molecule_atom_index_0_dist_std_div'] = df[f'molecule_atom_index_0_dist_std'] / df['dist']\n    df[f'molecule_atom_index_1_dist_mean'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('mean')\n    df[f'molecule_atom_index_1_dist_mean_diff'] = df[f'molecule_atom_index_1_dist_mean'] - df['dist']\n    df[f'molecule_atom_index_1_dist_mean_div'] = df[f'molecule_atom_index_1_dist_mean'] / df['dist']\n    df[f'molecule_atom_index_1_dist_max'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('max')\n    df[f'molecule_atom_index_1_dist_max_diff'] = df[f'molecule_atom_index_1_dist_max'] - df['dist']\n    df[f'molecule_atom_index_1_dist_max_div'] = df[f'molecule_atom_index_1_dist_max'] / df['dist']\n    df[f'molecule_atom_index_1_dist_min'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('min')\n    df[f'molecule_atom_index_1_dist_min_diff'] = df[f'molecule_atom_index_1_dist_min'] - df['dist']\n    df[f'molecule_atom_index_1_dist_min_div'] = df[f'molecule_atom_index_1_dist_min'] / df['dist']\n    df[f'molecule_atom_index_1_dist_std'] = df.groupby(['molecule_name', 'atom_index_1'])['dist'].transform('std')\n    df[f'molecule_atom_index_1_dist_std_diff'] = df[f'molecule_atom_index_1_dist_std'] - df['dist']\n    df[f'molecule_atom_index_1_dist_std_div'] = df[f'molecule_atom_index_1_dist_std'] / df['dist']\n    df[f'molecule_atom_1_dist_mean'] = df.groupby(['molecule_name', 'atom_1'])['dist'].transform('mean')\n    df[f'molecule_atom_1_dist_min'] = df.groupby(['molecule_name', 'atom_1'])['dist'].transform('min')\n    df[f'molecule_atom_1_dist_min_diff'] = df[f'molecule_atom_1_dist_min'] - df['dist']\n    df[f'molecule_atom_1_dist_min_div'] = df[f'molecule_atom_1_dist_min'] / df['dist']\n    df[f'molecule_atom_1_dist_std'] = df.groupby(['molecule_name', 'atom_1'])['dist'].transform('std')\n    df[f'molecule_atom_1_dist_std_diff'] = df[f'molecule_atom_1_dist_std'] - df['dist']\n    df[f'molecule_type_0_dist_std'] = df.groupby(['molecule_name', 'type_0'])['dist'].transform('std')\n    df[f'molecule_type_0_dist_std_diff'] = df[f'molecule_type_0_dist_std'] - df['dist']\n    df[f'molecule_type_dist_mean'] = df.groupby(['molecule_name', 'type'])['dist'].transform('mean')\n    df[f'molecule_type_dist_mean_diff'] = df[f'molecule_type_dist_mean'] - df['dist']\n    df[f'molecule_type_dist_mean_div'] = df[f'molecule_type_dist_mean'] / df['dist']\n    df[f'molecule_type_dist_max'] = df.groupby(['molecule_name', 'type'])['dist'].transform('max')\n    df[f'molecule_type_dist_min'] = df.groupby(['molecule_name', 'type'])['dist'].transform('min')\n    df[f'molecule_type_dist_std'] = df.groupby(['molecule_name', 'type'])['dist'].transform('std')\n    df[f'molecule_type_dist_std_diff'] = df[f'molecule_type_dist_std'] - df['dist']\n\n    return df\n\ntrain = reduce_mem_usage(create_more_features(train)).fillna(0)\ntest = reduce_mem_usage(create_more_features(test)).fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# transform categorical columns\ndef transform_onehot(df, col_name):\n    enc = OneHotEncoder()\n    arr = enc.fit_transform(df[col_name].values.reshape(-1, 1)).toarray().astype(int)\n    col_names = [col_name+'_'+cat for cat in enc.categories_[0]]\n    new_col = pd.DataFrame(arr, columns=col_names)\n    ret = df.join(new_col)\n    return ret.drop(columns=col_name)\n\ntrain = transform_onehot(train, 'atom_0')\ntrain = transform_onehot(train, 'atom_1')\n\ntest = transform_onehot(test, 'atom_0')\ntest = transform_onehot(test, 'atom_1')\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# preprocessing is ended, then start the regression\n##############\n# i don't need reference columns anymore\ntrain.drop(columns=['id', 'molecule_name', 'atom_0_H'], inplace=True)\ntest.drop(columns=['molecule_name', 'atom_0_H'], inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I'm gonna use three of regressors Adaboost, gradient boosting and xgboost.  \nSetting all features to training set is inappropriate, so i arranged top-15 importances of features on every type-regressors.  \n(Especially, we can build 8 models on every single atom type.)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Finding important features in order\ntype_model = dict()\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'KFold training started for {type_} model.')\n    X_train = train.loc[train['type'] == type_].drop(columns=['type', 'scalar_coupling_constant'])\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    folds = KFold(n_splits=5, shuffle=True)\n    X_train_sub = resample(X_train, n_samples=int(len(X_train)/1000), random_state=13)\n    y_train_sub = resample(y_train, n_samples=int(len(X_train)/1000), random_state=13)\n    importances = pd.Series(index=X_train.columns).fillna(0)\n    for i, (idx_train, idx_test) in enumerate(folds.split(X_train_sub.values, y_train_sub.values)):\n        print(f'{i+1} / 5 folds')\n        X_train_folds = X_train_sub.iloc[idx_train]\n        y_train_folds = y_train_sub.iloc[idx_train]\n        X_valid_folds = X_train_sub.iloc[idx_train]\n        y_valid_folds = y_train_sub.iloc[idx_train]\n        \n        # using 3 regression algoritms\n        #abr = AdaBoostRegressor(n_estimators=50, learning_rate=0.1, loss='exponential')\n        #gbr = GradientBoostingRegressor(n_estimators=50, learning_rate=0.1, max_depth=10, criterion='mae')\n        xgb_ = xgb.XGBRegressor(n_estimators=50, learning_rate=0.1, max_depth=10)\n        \n        #abr.fit(X_train_folds, y_train_folds)\n        #gbr.fit(X_train_folds, y_train_folds)\n        xgb_.fit(X_train_folds, y_train_folds)\n        \n        #importances += abr.feature_importances_\n        #importances += gbr.feature_importances_\n        importances += xgb_.feature_importances_\n        \n    type_model[type_] = importances","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The importance orders are diffrent from each other."},{"metadata":{"trusted":true},"cell_type":"code","source":"type_model['3JHN'].sort_values(ascending=False)[:15].sort_values().plot(kind='barh')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type_model['2JHC'].sort_values(ascending=False)[:15].sort_values().plot(kind='barh')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"important_features = dict()\nfor model, values in type_model.items():\n    important_features[model] = values.sort_values(ascending=False)[:15].index\n\nimportant_features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Looking for better models randomly."},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"#Random grid searching with XGBRegressor\nxgb_b = dict()\nxgb_score = 0\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'XGBoost Regressor : Random searching stated for {type_} model.')\n    X_train = train.loc[train['type'] == type_, important_features[type_]]\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    X_train_sub = resample(X_train, n_samples=int(len(X_train)/1000), random_state=43)\n    y_train_sub = resample(y_train, n_samples=int(len(X_train)/1000), random_state=43)\n    params={'learning_rate': [0.01, 0.1, 1.0],\n                    'min_child_weight': [1, 5, 10],\n                    'n_estimators': [50, 100, 200, 400, 800],\n                    'max_depth': [4, 10, 15, 20],\n                    'gamma': [0.5, 1, 1.5, 2, 5],\n                    'subsample': [0.1, 0.5, 1.0],\n                    'colsample_bytree': [0.1, 0.5, 1.0],\n                    'colsample_bylevel': [0.1, 0.5, 1.0],\n                    'scale_pos_weight': [0.01, 0.1, 1.0],\n                    'reg_lambda': [0.1, 1, 10],\n                    'reg_alpha': [0.01, 0.1, 1.0],\n                    'max_delta_step': [0, 1, 10],\n                    'scale_pos_weight': [0.01, 0.1, 1]}\n    cv_xgb = RandomizedSearchCV(estimator=xgb.XGBRegressor(objective='reg:squarederror'),\n                                   param_distributions=params,\n                                   cv=KFold(n_splits=5, shuffle=True),\n                                scoring='neg_mean_absolute_error',\n                                   n_iter=15,\n                                   verbose=1)\n    cv_xgb.fit(X_train_sub.values, y_train_sub.values)\n    xgb_b[type_] = cv_xgb.best_estimator_\n    xgb_score += cv_xgb.best_score_\n    \n\"\"\"\n#Grid searching with Adaboost regressor\nabr_b = dict()\nabr_score = 0\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'Adaboost Regressor : Random searching started for {type_} model.')\n    X_train = train.loc[train['type'] == type_, important_features[type_]]\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    X_train_sub = resample(X_train, n_samples=int(len(X_train)/1000), random_state=23)\n    y_train_sub = resample(y_train, n_samples=int(len(X_train)/1000), random_state=23)\n    params={'n_estimators': [50, 100, 200, 400, 800],\n            'learning_rate':[0.01, 0.1, 1],\n            'loss': ['linear', 'square', 'exponential']\n            }\n    cv_abr = RandomizedSearchCV(estimator=AdaBoostRegressor(),\n                                   param_distributions=params,\n                                   cv=KFold(n_splits=5, shuffle=True),\n                                scoring='neg_mean_absolute_error',\n                                   n_iter=15,\n                                   verbose=1)\n    cv_abr.fit(X_train_sub.values, y_train_sub.values)\n    abr_b[type_] = cv_abr.best_estimator_\n    abr_score += cv_abr.best_score_\n\n\n#Random grid searching with GradientBoosting regressor\ngbr_b = dict()\ngbr_score = 0\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'Gradient Boosting Regressor : Random searching stated for {type_} model.')\n    X_train = train.loc[train['type'] == type_, important_features[type_]]\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    X_train_sub = resample(X_train, n_samples=int(len(X_train)/1000), random_state=33)\n    y_train_sub = resample(y_train, n_samples=int(len(X_train)/1000), random_state=33)\n    params={'n_estimators': [50, 100, 200, 400, 800],\n            'learning_rate':[0.01, 0.1, 1],\n            'loss': ['ls', 'lad', 'huber', 'quantile'],\n            'subsample' : [0.01, 0.1, 1],\n            'min_samples_split': [0.01, 0.1, 2],\n            'min_samples_leaf': [0.01, 0.1, 2],\n            'max_depth': [4, 10, 15, 20],\n            'max_features':['auto', 'sqrt', 'log2']\n            }\n    cv_gbr = RandomizedSearchCV(estimator=GradientBoostingRegressor(criterion='mae'),\n                                   param_distributions=params,\n                                   cv=KFold(n_splits=5, shuffle=True),\n                                scoring='neg_mean_absolute_error',\n                                   n_iter=15,\n                                   verbose=1)\n    cv_gbr.fit(X_train_sub.values, y_train_sub.values)\n    gbr_b[type_] = cv_gbr.best_estimator_\n    gbr_score += cv_gbr.best_score_\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n# Training and prediction with XGB regressor\nxgb_predicted = pd.Series(index=test.index)\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'{type_} type training started.')\n    X_train = train.loc[train['type'] == type_, important_features[type_]]\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    X_test = test.loc[test['type'] == type_, important_features[type_]]\n    clf_xgb = xgb_b[type_]\n    clf_xgb.fit(X_train.values, y_train.values)\n    result = pd.Series(data=clf_xgb.predict(X_test.values), index=X_test.index)\n    xgb_predicted = xgb_predicted.add(result, fill_value=0)\n\n# Training and prediction with AdaBoost regressor\nabr_predicted = pd.Series(index=test.index)\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'{type_} type training started.')\n    X_train = train.loc[train['type'] == type_, important_features[type_]]\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    X_test = test.loc[test['type'] == type_, important_features[type_]]\n    clf_abr = abr_b[type_]\n    clf_abr.fit(X_train.values, y_train.values)\n    result = pd.Series(data=clf_abr.predict(X_test.values), index=X_test.index)\n    abr_predicted = abr_predicted.add(result, fill_value=0)\n\n# Training and prediction with GradientBoosting regressor\ngbr_predicted = pd.Series(index=test.index)\nfor type_ in ['1JHC', '2JHH', '1JHN', '2JHN', '2JHC', '3JHH', '3JHC', '3JHN']:\n    print(f'{type_} type training started.')\n    X_train = train.loc[train['type'] == type_, important_features[type_]]\n    y_train = train.loc[train['type'] == type_, 'scalar_coupling_constant']\n    X_test = test.loc[test['type'] == type_, important_features[type_]]\n    clf_gbr = gbr_b[type_]\n    clf_gbr.fit(X_train.values, y_train.values)\n    result = pd.Series(data=clf_gbr.predict(X_test.values), index=X_test.index)\n    gbr_predicted = gbr_predicted.add(result, fill_value=0)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\n# average the predicted target values\n#predicted = (xgb_predicted + abr_predicted + gbr_predicted) / 3\n\n# make a submission file\n#pd.DataFrame({'id':test['id'], 'scalar_coupling_constant':predicted}).to_csv('predicted.csv', index=False)\n#pd.DataFrame({'id':test['id'], 'scalar_coupling_constant':abr_predicted}).to_csv('abr_predicted.csv', index=False)\n#pd.DataFrame({'id':test['id'], 'scalar_coupling_constant':gbr_predicted}).to_csv('gbr_predicted.csv', index=False)\npd.DataFrame({'id':test['id'], 'scalar_coupling_constant':xgb_predicted}).to_csv('xgb_predicted.csv', index=False)\n\"\"\"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.7"}},"nbformat":4,"nbformat_minor":1}