{"cells":[{"metadata":{},"cell_type":"markdown","source":"## This kernel shows up how to achieve -1.57 score on LB only on distance features using Extremely Randomized Trees.\n\nCore ideas:\n- basic features (63)\n  - distances features between atom_0 and atom_1 encoded with atom type (3)\n  - neighbor atoms distance features:\n    - top5 of distances from atom_0 and atom_1 to nearest neighbors (10) - `med2` datasets\n      - column format: `d{atom_index}_med2_neighbor{1..5}`\n    - top5 of distances from atom_0 and atom_1 to nearest neighbors using atom type (50) - `sep2` datasets\n      - column format: `d{atom_index}_typ{atom_type}_sep2_neighbor{1..5}`\n- boruta feature selection (63 -> 37)\n  - some distances to nearest neighbor atoms using atom type was dropped\n- ExtraTreeRegressor * 8 for different interactions\n  - more trees can be used for interactions with small count of samples\n  - scikit-optimize with GP (gp_minimize)\n    - competition metric\n    - hold-out validation\n\nThis approach allowed to achieve -1.57 score on LB"},{"metadata":{"trusted":true},"cell_type":"code","source":"EXP_NUMBER = 63","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data loading"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = pd.read_csv('../input/champs-scalar-coupling/train.csv')\ndf_test = pd.read_csv('../input/champs-scalar-coupling/test.csv')\ndf_struct = pd.read_csv('../input/champs-scalar-coupling/structures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape, df_test.shape, df_struct.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import LabelEncoder","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"type_enc = LabelEncoder()\n\ndf_train['type'] = type_enc.fit_transform(df_train['type'])\ndf_test['type'] = type_enc.transform(df_test['type'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"atom_enc = LabelEncoder()\n\ndf_struct['atom'] = atom_enc.fit_transform(df_struct['atom'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(df_struct, \n                          left_on=['molecule_name','atom_index_0'], \n                          right_on=['molecule_name','atom_index'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(df_struct, \n                          left_on=['molecule_name','atom_index_1'], \n                          right_on=['molecule_name','atom_index'],\n                          suffixes=('_0', '_1'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_neighbors = 5\n\nneigh_da_cols = [f'da_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1) for typ in range(5)]\nneigh_d0_cols = [f'd0_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1) for typ in range(5)]\nneigh_d1_cols = [f'd1_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1) for typ in range(5)]\nneigh_cosa_cols = [f'cosine_da_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1) \n                   for typ in range(5)]\nneigh_cos0_cols = [f'cosine_d0_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1) \n                   for typ in range(5)]\nneigh_cos1_cols = [f'cosine_d1_typ{typ}_sep2_neighbor{i}' for i in range(1, n_neighbors+1) \n                   for typ in range(5)]\nneigh_all_cols = neigh_d0_cols + neigh_d1_cols # + neigh_da_cols\n#                   neigh_cosa_cols\n#                   neigh_cos0_cols\n#                   neigh_cos1_cols\nneigh_cols = ['id'] + neigh_all_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_frames(files, columns, parent_folder='../input/dist-feats'):\n    df_neigh = pd.DataFrame()\n    for name in files:\n        print(f'Loading {name} ...')\n        dfn = pd.read_csv(f'{parent_folder}/{name}', usecols=columns)\n        df_neigh = pd.concat([df_neigh, dfn])\n        del(dfn)\n    return df_neigh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_neigh_files = [\n'top-5_atoms-[\\'sep2\\']_count-1164536_44c3ddd5-0dd4-42b7-acac-d0a0e7bd444e.csv',\n'top-5_atoms-[\\'sep2\\']_count-1164536_7e41a7b8-d1ed-4d11-be5e-5166d347a8a2.csv',\n'top-5_atoms-[\\'sep2\\']_count-1164536_e2e300a0-55c4-453a-b40d-b62e4cf7b336.csv',\n'top-5_atoms-[\\'sep2\\']_count-1164539_42ba6898-a223-4a78-8157-b1f015c09806.csv',\n]\n\ndf_neigh = load_frames(df_train_neigh_files, neigh_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(df_neigh, on='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_neighbors = 5\n\nneigh_dm_cols_2 = [f'dm_med2_neighbor{i}' for i in range(1, n_neighbors+1)]\nneigh_d0_cols_2 = [f'd0_med2_neighbor{i}' for i in range(1, n_neighbors+1)]\nneigh_d1_cols_2 = [f'd1_med2_neighbor{i}' for i in range(1, n_neighbors+1)]\nneigh_cos0_cols_2 = [f'cos0_med2_neighbor{i}' for i in range(1, n_neighbors+1)]\nneigh_all_cols_2 = neigh_d0_cols_2 + neigh_d1_cols_2 # + neigh_dm_cols_2 + neigh_cos0_cols_2\nneigh_cols_2 = ['id'] + neigh_all_cols_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_neigh_files = [\n'top-5_atoms-[\\'med2\\']_count-1164536_0a918683-2fe3-43a1-b239-c900712c1433.csv',\n'top-5_atoms-[\\'med2\\']_count-1164536_b845e873-855d-424b-a7f3-50d89824b0ea.csv',\n'top-5_atoms-[\\'med2\\']_count-1164536_df446883-d0af-486c-877b-f8d834c0b00f.csv',\n'top-5_atoms-[\\'med2\\']_count-1164539_de6e2745-1a72-490c-ac29-69ad07c56910.csv',\n]\n\ndf_neigh = load_frames(df_train_neigh_files, neigh_cols_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(df_neigh, on='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.set_index('id', inplace=True)\ndf_train.sort_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['atom_0'].value_counts(), df_train['atom_1'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.drop(['atom_0'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.merge(df_struct, \n                        left_on=['molecule_name','atom_index_0'], \n                        right_on=['molecule_name','atom_index'],\n                        sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.merge(df_struct, \n                        left_on=['molecule_name','atom_index_1'], \n                        right_on=['molecule_name','atom_index'],\n                        suffixes=('_0', '_1'),\n                        sort=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_neigh_files = [\n'top-5_atoms-[\\'sep2\\']_count-626385_7623bd50-41b8-4c44-b40a-e37837700460.csv',\n'top-5_atoms-[\\'sep2\\']_count-626385_878ae0ca-67c7-491e-b580-fbdc384c2477.csv',\n'top-5_atoms-[\\'sep2\\']_count-626385_e4f75a2d-302c-476b-b60c-c5624925ef01.csv',\n'top-5_atoms-[\\'sep2\\']_count-626387_4792052c-d539-4e62-8e37-c4ecf81ef669.csv',\n]\n\ndf_neigh = load_frames(df_test_neigh_files, neigh_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.merge(df_neigh, on='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test_neigh_files = [\n'top-5_atoms-[\\'med2\\']_count-626385_0072225a-03f6-42bf-b1ed-b81be41c6d14.csv',\n'top-5_atoms-[\\'med2\\']_count-626385_7ed7d305-6c6a-4d3c-a6e8-f8e274fd461f.csv',\n'top-5_atoms-[\\'med2\\']_count-626385_b4f9dd09-7c61-42a7-9534-85a31496d1a5.csv',\n'top-5_atoms-[\\'med2\\']_count-626387_a9d566a5-1248-4341-abec-b71a87bbe21d.csv',\n]\n\ndf_neigh = load_frames(df_test_neigh_files, neigh_cols_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.merge(df_neigh, on='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.set_index('id', inplace=True)\ndf_test.sort_index(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test['atom_0'].value_counts(), df_test['atom_1'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = df_test.drop(['atom_0'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(df_struct)\ndel(df_neigh)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Distance"},{"metadata":{"trusted":true},"cell_type":"code","source":"def calc_polar(df, suffix_0='_0', suffix_1='_1', angles=True):\n    '''Calculate polar coords\n    '''\n    dx = df['x'+suffix_0] - df['x'+suffix_1]\n    dy = df['y'+suffix_0] - df['y'+suffix_1]\n    dz = df['z'+suffix_0] - df['z'+suffix_1]\n    if angles:\n        dx2 = np.power(dx, 2)\n        dy2 = np.power(dy, 2)\n        dz2 = np.power(dz, 2)\n        d = np.sqrt(dx2 + dy2 + dz2)\n        az = np.arctan2(dy, dx)\n        el = np.arctan2(dz, np.sqrt(dx2 + dy2))\n        return d, az, el\n    else:\n        return np.sqrt(dx**2 + dy**2 + dz**2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def feat_eng(df):\n    d, _, _ = calc_polar(df)\n    df_dummy = pd.get_dummies(df[['atom_1']], columns=['atom_1'])\n    atom_1_cols = [x for x in df_dummy.columns if 'atom_1' in x]\n    d_cols = [f'd_{x[7:]}' for x in atom_1_cols]\n    df[d_cols] = df_dummy[atom_1_cols].multiply(d, axis='index')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat_eng(df_train)\nfeat_eng(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ncorr = df_train.corr()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(10,10))\nplt.matshow(corr, fignum=1, cmap='coolwarm')\nplt.xticks(range(len(corr.columns)), corr.columns, rotation='vertical');\nplt.yticks(range(len(corr.columns)), corr.columns)\nplt.colorbar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(corr)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data splits preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = 'scalar_coupling_constant'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"d_cols = [x for x in df_train.columns if x.startswith('d_') and len(x) == 3]\nfeat_cols = d_cols + neigh_all_cols + neigh_all_cols_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_types = df_train['type'].values\ntrain_types.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = df_train[feat_cols].values\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = df_train[target].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_types = df_test['type'].values\ntest_types.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = df_test[feat_cols].values\nX_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del(df_train)\ndel(df_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom scipy.stats import ks_2samp\n\n\ndef split(X, y, types, random_state=1, p_value=0.9, test_size=0.2, verbose=True):\n    X_train, X_test, y_train, y_test, train_types, test_types = None, None, None, None, None, None\n\n    while True:\n        X_train, X_test, y_train, y_test, train_types, test_types = train_test_split(\n            X, y, types, \n            test_size=test_size, \n            stratify=types, \n            random_state=random_state\n        )\n        if p_value is None:\n            return X_train, X_test, y_train, y_test, train_types, test_types\n        st1 = ks_2samp(y, y_train)\n        st2 = ks_2samp(y, y_test)\n        st3 = ks_2samp(y_train, y_test)\n        if verbose:\n            print('RS =', random_state)\n            print(st1)\n            print(st2)\n            print(st3)\n        if st1.pvalue > p_value and st2.pvalue > p_value and st3.pvalue > p_value:\n            return X_train, X_test, y_train, y_test, train_types, test_types\n        del(X_train)\n        del(X_test)\n        del(y_train)\n        del(y_test)\n        del(train_types)\n        del(test_types)\n        random_state += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_part, X_valid, y_train_part, y_valid, train_part_types, valid_types = split(X_train, y_train, train_types, \n                                                                                    random_state=42, p_value=0.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_valid.shape, y_valid.shape, valid_types.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_small, _, y_train_small, _, train_small_types, _ = split(X_train, y_train, train_types, test_size=0.90) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_small.shape, y_train_small.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape, X_train_part.shape, X_valid.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature selection"},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"# %%time\n# from sklearn.ensemble import RandomForestRegressor\n# from boruta import BorutaPy\n\n# reg = RandomForestRegressor(n_estimators=12, n_jobs=-1, random_state=42)\n# fs = BorutaPy(reg, max_iter=10, n_estimators=12, verbose=2, random_state=42)\n\n# fs.fit(X_train_small, y_train_small)  \n\n# feat_cols_dropped = sorted([feat_cols[i] for i in range(len(feat_cols)) if not fs.support_[i]])\n# print('Feats dropped:', len(feat_cols_dropped))\n\n# feat_cols_selected = sorted([feat_cols[i] for i in range(len(feat_cols)) if fs.support_[i]])\n# print('Feats selected:', len(feat_cols_selected))\n\n# feat_cols_selected\n\nfeat_cols_selected = [\n 'd0_med2_neighbor1',\n 'd0_med2_neighbor2',\n 'd0_med2_neighbor3',\n 'd0_med2_neighbor4',\n 'd0_med2_neighbor5',\n 'd0_typ0_sep2_neighbor1',\n 'd0_typ0_sep2_neighbor2',\n 'd0_typ0_sep2_neighbor3',\n 'd0_typ0_sep2_neighbor4',\n 'd0_typ0_sep2_neighbor5',\n 'd0_typ2_sep2_neighbor1',\n 'd0_typ2_sep2_neighbor2',\n 'd0_typ2_sep2_neighbor3',\n 'd0_typ2_sep2_neighbor4',\n 'd0_typ2_sep2_neighbor5',\n 'd0_typ3_sep2_neighbor1',\n 'd0_typ4_sep2_neighbor1',\n 'd1_med2_neighbor1',\n 'd1_med2_neighbor2',\n 'd1_med2_neighbor3',\n 'd1_med2_neighbor4',\n 'd1_med2_neighbor5',\n 'd1_typ0_sep2_neighbor1',\n 'd1_typ0_sep2_neighbor2',\n 'd1_typ0_sep2_neighbor3',\n 'd1_typ0_sep2_neighbor4',\n 'd1_typ0_sep2_neighbor5',\n 'd1_typ2_sep2_neighbor1',\n 'd1_typ2_sep2_neighbor2',\n 'd1_typ2_sep2_neighbor3',\n 'd1_typ2_sep2_neighbor4',\n 'd1_typ2_sep2_neighbor5',\n 'd1_typ3_sep2_neighbor1',\n 'd1_typ4_sep2_neighbor1',\n 'd_0',\n 'd_2',\n 'd_3'\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_train = fs.transform(X_train)\n# X_train_part = fs.transform(X_train_part)\n# X_valid = fs.transform(X_valid)\n# X_train_small = fs.transform(X_train_small)\n# X_test = fs.transform(X_test)\n\nfeat_cols_selected_indices = [idx for idx, name in enumerate(feat_cols) if name in feat_cols_selected]\nX_train = X_train[:, feat_cols_selected_indices]\nX_train_part = X_train_part[:, feat_cols_selected_indices]\nX_valid = X_valid[:, feat_cols_selected_indices]\nX_train_small = X_train_small[:, feat_cols_selected_indices]\nX_test = X_test[:, feat_cols_selected_indices]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## One model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.ensemble import ExtraTreesRegressor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg = ExtraTreesRegressor(n_estimators=10, n_jobs=-1, random_state=42, verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nreg.fit(X_train_part, y_train_part)","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"y_pred = reg.predict(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def metric(y, y_hat, types, type_filter=None, verbose=True):\n    res = 0\n    uniq_types = np.unique(types)\n    if verbose:\n        print('typ|   cnt(%)   |    sum     |  log')\n        cnt_total = types.shape[0]\n    for typ in uniq_types:\n        if type_filter is not None and type_filter != typ:\n            continue\n        idx = np.where(types == typ)[0]\n        cnt = idx.shape[0]\n        res_typ = np.sum(np.abs(y[idx] - y_hat[idx])) / cnt\n        assert cnt == y[idx].shape[0] == y_hat[idx].shape[0], 'inconsistent idx'\n        if verbose:\n            print(f'{typ:3}|{cnt:8}({1+cnt*100//cnt_total:02d})|{res_typ:12.9}|{np.log(res_typ)}')\n        res += np.log(res_typ)\n    uniq_types_cnt = uniq_types.shape[0] if type_filter is None else 1\n    res /= uniq_types_cnt   \n    if verbose:\n        print(f'Result: {res} for {uniq_types_cnt} types')\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"metric(y_valid, y_pred, valid_types)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Hyperparams optimization"},{"metadata":{"trusted":true},"cell_type":"code","source":"from skopt.space import Real, Integer, Categorical","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# The list of hyper-parameters we want to optimize. For each one we define the bounds,\n# the corresponding scikit-learn parameter name, as well as how to sample values\n# from that dimension (`'log-uniform'` for the learning rate)\nspace = [\n    Integer(10, 100, name='n_estimators'),\n#     Integer(2, 100, name='min_samples_split'),  # always max\n#     Integer(1, 100, name='min_samples_leaf'),   # always min\n#     Categorical([True, False], name='bootstrap'),  # problem with plotting results\n    Real(0.1, 1.0, prior='uniform', name='max_features'),\n#     Integer(25, 125, name='max_depth'),  # always max\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_estimators_by_type = {\n    0: 32, 1: 160, 2: 28, 3: 100, 4: 160, 5: 28, 6: 32, 7: 160\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\nfrom lightgbm import LGBMRegressor\nfrom tqdm import tqdm_notebook\n\n\ndef get_estimator(estimator_model='rf', n_estimators=10, verbose=False):\n    if estimator_model == 'rf':\n        cls = RandomForestRegressor\n    elif estimator_model == 'et':\n        cls = ExtraTreesRegressor\n    elif estimator_model == 'lgbm':\n        cls = LGBMRegressor\n    else:\n        return None\n    print('Selected estimator class:', cls)\n    return cls(n_estimators=n_estimators, n_jobs=-1, random_state=42, verbose=verbose)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import cross_val_score\nfrom skopt.space import Real, Integer, Categorical\nfrom skopt.utils import use_named_args\nfrom skopt import gp_minimize\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook\n\n\ndef skopt_hyper(space, n_estimators_by_type, \n                X_train, y_train, train_types, \n                X_valid, y_valid, valid_types,\n                estimator_model='rf', n_calls=30, \n                random_state=42, verbose=True):\n    \n    reg = get_estimator(estimator_model, verbose=0)  # False\n    if reg is None:\n        return None\n\n    res_opts = {}\n    for typ in tqdm_notebook(np.unique(train_types)):\n\n        # This decorator allows your objective function to receive the parameters as\n        # keyword arguments. This is particularly convenient when you want to set scikit-learn\n        # estimator parameters\n        @use_named_args(space)\n        def objective(**params):\n            reg.set_params(**params)\n            mask = train_types == typ\n    #         scores = cross_val_score(reg, X_train[mask], y_train[mask], cv=3, n_jobs=-1,\n    #                                  scoring=\"neg_mean_absolute_error\")\n    #         return -np.mean(scores)\n            reg.fit(X_train[mask], y_train[mask])\n            y_valid_pred = reg.predict(X_valid)\n            score = metric(y_valid, y_valid_pred, valid_types, typ, verbose=verbose)\n            return score\n\n        if n_estimators_by_type is not None:\n            space[0] = Integer(10, n_estimators_by_type[typ], name='n_estimators')\n        if verbose:\n            print(f'--- Type {typ}, space {space} ---')\n            \n        res_opt = gp_minimize(objective, space, n_calls=n_calls, \n                              verbose=True, random_state=random_state)\n        res_opts[typ] = res_opt\n        if verbose:\n            print(f'--- optimal params: {res_opt.x} ---')\n\n    from skopt.plots import plot_convergence, plot_evaluations, plot_objective\n    for typ in np.unique(train_types):\n        if verbose:\n            print('Type', typ)\n        res_opt = res_opts[typ]\n        plot_convergence(res_opt)\n        plot_evaluations(res_opt)\n        plot_objective(res_opt)\n        plt.show()\n       \n    return {k: v.x for k, v in res_opts.items()}","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%time\n# res_opts = skopt_hyper(space, n_estimators_by_type, \n#                        X_train_small, y_train_small, train_small_types,\n#                        X_valid, y_valid, valid_types, \n#                        estimator_model='et', verbose=True)\nres_opts = {\n 0: [32, 0.9356368713812235],\n 1: [160, 0.9226985331078089],\n 2: [28, 0.978079768528465],\n 3: [100, 1.0],\n 4: [160, 1.0],\n 5: [28, 0.9688476690205474],\n 6: [32, 1.0],\n 7: [160, 0.9976351777608269]\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res_opts","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Models by interactions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_predict_sep(X_train, y_train, X_train_part, y_train_part, X_valid, y_valid, \n                          train_types, train_part_types, valid_types, \n                          X_test=None, test_types=None,\n                          estimator_model='rf', cnt_estimators=None, res_opts=None, params=None):\n    assert X_train.shape[0] == y_train.shape[0] == train_types.shape[0], 'Inconsistent train - X and y'\n    assert X_train_part.shape[0] == y_train_part.shape[0] == train_part_types.shape[0], 'Inconsistent train_part - X and y'\n    assert X_valid.shape[0] == y_valid.shape[0] == valid_types.shape[0], 'Inconsistent valid - X and y'\n    y_pred_valid = np.empty(X_valid.shape[0])\n    y_pred_test = None\n    if X_test is not None and test_types is not None:\n        assert X_test.shape[0] == test_types.shape[0], 'Inconsistent train - X and y'\n        y_pred_test = np.empty(X_test.shape[0])\n    typ_cnt = np.unique(train_types).shape[0]\n    if cnt_estimators is None:\n        cnt_estimators = [10] * typ_cnt\n    assert len(cnt_estimators) == typ_cnt, 'Inconsistent estimators count'\n    for n_estimators, typ in tqdm_notebook(zip(cnt_estimators, range(typ_cnt)), total=len(cnt_estimators)):\n        print('+++++ Fitting model for typ', typ)\n        mask = train_part_types == typ\n        reg = get_estimator(estimator_model, cnt_estimators, verbose=2)\n        if reg is None:\n            return None\n        if params is not None:\n            reg.set_params(**params[typ])\n        elif res_opts is not None:\n            res_opt = res_opts[typ]\n            reg_params = {\n                'n_estimators':      res_opt[0],\n                'max_features':      res_opt[1],\n            }\n            reg.set_params(**reg_params)\n        print(reg)    \n        reg.fit(X_train_part[mask], y_train_part[mask])\n        print('+++++ Predicting with model for typ', typ)\n        mask = valid_types == typ\n        y_pred_valid[mask] = reg.predict(X_valid[mask])\n        metric(y_valid, y_pred_valid, valid_types, typ)\n        if X_test is not None and test_types is not None:\n            print('+++++ Fitting model for typ', typ)\n            mask = train_types == typ\n            reg.fit(X_train[mask], y_train[mask])\n            print('+++++ Predicting with model for typ', typ)\n            mask = test_types == typ\n            y_pred_test[mask] = reg.predict(X_test[mask])\n    print('+++++ Overall validation report')\n    metric(y_valid, y_pred_valid, valid_types)\n    return y_pred_test, y_pred_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt_estimators = None","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":true},"cell_type":"code","source":"%%time\ny_pred, _ = train_and_predict_sep(X_train, y_train, X_train_part, y_train_part, X_valid, y_valid, \n                                  train_types, train_part_types, valid_types, \n                                  X_test=X_test, test_types=test_types,\n                                  estimator_model='et', cnt_estimators=cnt_estimators, res_opts=res_opts)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission output"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(y_pred, bins=100)\npass","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_subm = pd.read_csv('../input/champs-scalar-coupling/sample_submission.csv', index_col='id')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_subm[target] = y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_subm.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fn_csv = f'./{EXP_NUMBER}.subm.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_subm.to_csv(fn_csv)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}