{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport scipy as sp\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn.linear_model import LinearRegression, Lasso, Ridge\nfrom sklearn.preprocessing import OneHotEncoder, StandardScaler\nfrom sklearn.model_selection import KFold, cross_validate, train_test_split, GridSearchCV\nfrom sklearn.metrics import mean_squared_error, r2_score\nfrom sklearn.kernel_ridge import KernelRidge\nfrom sklearn.ensemble import RandomForestRegressor\nimport os\n\n%matplotlib inline\n\nimport warnings\nwarnings.simplefilter(action = 'ignore', category = FutureWarning)\n\nZIP_DIR = './Data/'\nSEED = 42","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def result(model, x_train, x_test):\n    train_prediction = model.predict(x_train)\n    test_prediction = model.predict(x_test)\n\n    results = pd.DataFrame({\n                        'MSE_train':[mean_squared_error(y_train, train_prediction)],\n                        'MSE_test':[mean_squared_error(y_test, test_prediction)],\n                        'R2_train':[r2_score(y_train, train_prediction)],\n                        'R2_test':[r2_score(y_test, test_prediction)]\n                        })\n    return results","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_fancy_coefs(model):\n    fancy_coef = dict()\n    fancy_coef['coef-s model '] = list(model.coef_) + [model.intercept_]\n\n    coefs = pd.DataFrame.from_dict(fancy_coef)\n    coefs.index = list(x_train.columns.values) + ['intercept']\n    return coefs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_feature_importance(model):\n    importance = pd.DataFrame({\n                            'abs_weight': np.abs(model.coef_),\n                            'feature': x_train.columns.values\n    })\n    imp20 = importance.sort_values(by='abs_weight', ascending=False)[:20]\n    sns.barplot(y='feature', x='abs_weight', data=imp20, orient='h');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"file_folder = '../input/champs-scalar-coupling' if 'champs-scalar-coupling' in os.listdir('../input/') else '../input'\ntrain = pd.read_csv(f'{file_folder}/train.csv')\ntest = pd.read_csv(f'{file_folder}/test.csv')\nsub = pd.read_csv(f'{file_folder}/sample_submission.csv')\nstructures = pd.read_csv(f'{file_folder}/structures.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"structures.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Missing values"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Count of NaN values: \\n', train.isna().sum())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Пропуски в данных отсутствуют."},{"metadata":{},"cell_type":"markdown","source":"#### Feature Categorization"},{"metadata":{"trusted":true},"cell_type":"code","source":"ohe = OneHotEncoder(handle_unknown='ignore', sparse = False)\nnew_ohe_features = ohe.fit_transform(train['type'].values.reshape(-1,1))\ntmp = pd.DataFrame(new_ohe_features, columns = ['type' + str(i+1) for i in range(new_ohe_features.shape[1])])\ntrain = pd.concat([train, tmp],axis=1)\ntrain = train.drop(['type'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = pd.DataFrame(train['scalar_coupling_constant'].values.reshape(-1,1), columns = ['scalar_coupling_constant'])\nX = train.drop(['scalar_coupling_constant'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA"},{"metadata":{},"cell_type":"markdown","source":"#### Target variable analysis\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"figure = sns.distplot(y, norm_hist=True)\nfigure.set_title('scalar_coupling_constant distribution');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Распределение целевой переменной является бимодальным."},{"metadata":{},"cell_type":"markdown","source":"#### Correlation between features"},{"metadata":{"trusted":true},"cell_type":"code","source":"corr =train.corr()\ncorr_target = corr['scalar_coupling_constant']\ncond = np.abs(corr_target)>0.1\ntop_corr = corr.loc[cond, cond]\nsns.heatmap(top_corr, annot = True);","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Baseline Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"X = X.drop(['id', 'molecule_name'], axis=1)\ntest = test.drop(['id', 'molecule_name'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(X,y, test_size = 0.3, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lin_reg = LinearRegression()\nlin_reg.fit(x_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = result(lin_reg, x_train, x_test)\nresults","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## L2-regularization"},{"metadata":{"trusted":true},"cell_type":"code","source":"ridge = Ridge()\nridge.fit(x_train, y_train);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = result(ridge, x_train, x_test)\nresults","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Результат остался прежним."},{"metadata":{},"cell_type":"markdown","source":"## L1-regularization"},{"metadata":{"trusted":true},"cell_type":"code","source":"lasso = Lasso()\nlasso.fit(x_train, y_train);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = result(lasso, x_train, x_test)\nresults","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Результат ухудшился."},{"metadata":{},"cell_type":"markdown","source":"## Random Forest"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nrfr = RandomForestRegressor()\n\nn_fold = 5\nfolds = KFold(n_splits=n_fold, shuffle=True, random_state=11)\n\nparameter_grid = {'n_estimators': [100, 300],\n                  'max_depth': [15, 50]\n                 }\n\ngrid_search = GridSearchCV(rfr, param_grid=parameter_grid, cv=folds, scoring='neg_mean_absolute_error', n_jobs=-1)\ngrid_search.fit(x_train, y_train)\nprint('Best score: {}'.format(grid_search.best_score_))\nprint('Best parameters: {}'.format(grid_search.best_params_))\n\nrfr = RandomForestRegressor(**grid_search.best_params_)\nresult_dict_rfr = train_model_regression(X=x_train, X_test=x_test, y=y_train, params=params, folds=folds, model_type='sklearn', model=rfr, eval_metric='mae', plot_feature_importance=False,\n                                                          verbose=10, early_stopping_rounds=100, n_estimators=500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = result(rfr, x_train, x_test)\nresults","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.0"}},"nbformat":4,"nbformat_minor":1}