{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#### <center> The prupose of this notebook is to show you some </center>\n# <center> Scikit-learn Algorithms 👌👌👌 </center>","metadata":{"execution":{"iopub.status.busy":"2022-02-21T04:01:15.304133Z","iopub.execute_input":"2022-02-21T04:01:15.306533Z","iopub.status.idle":"2022-02-21T04:01:15.313175Z","shell.execute_reply.started":"2022-02-21T04:01:15.306459Z","shell.execute_reply":"2022-02-21T04:01:15.311888Z"}}},{"cell_type":"markdown","source":"#### The workflow wil be the following:\n\n1. Install Intel's Scikit-learn accelerator\n2. Read data and reduce memory\n3. Fix train data\n - columns all NaN, bedrooms\n4. Simple impute to not lose that muhc data\n5. Subselect data\n - For this exercise, only numeric columns\n5. Train 5 estimators (adjusted for anti-overfitting) with CV\n - Linear regression\n - Simple Decision tree\n - Random Forest\n - Gradient Boosting\n - Histogram Boosting\n6. Select best estimator by score\n7. Predict on test data\n\nTODO: Hyperparameter search...","metadata":{}},{"cell_type":"markdown","source":"### DISCUSSION: \n\n#### As you can see from the simple plot, there is a HUGE price in the training dataset. This will make the (root) mean squared error of any estimator big, if there is a similar case in the test data...\n\n#### The importance of iid data...\n\n#### Anyways, I hope this notebook is helpful to you.\n\n#### Any recomendations or comments, let me know ok.","metadata":{}},{"cell_type":"code","source":"%reset -sf\n\nfrom IPython.core.interactiveshell import InteractiveShell\nInteractiveShell.ast_node_interactivity = 'all'","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:05.518232Z","iopub.execute_input":"2022-02-24T03:42:05.51869Z","iopub.status.idle":"2022-02-24T03:42:05.884999Z","shell.execute_reply.started":"2022-02-24T03:42:05.518634Z","shell.execute_reply":"2022-02-24T03:42:05.883897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install -U scikit-learn\n\n# Patch Xeon Intel OneAPI Scikit accelerator\n!pip install scikit-learn-intelex\nfrom sklearnex import patch_sklearn\npatch_sklearn()","metadata":{"_kg_hide-output":true,"execution":{"iopub.status.busy":"2022-02-24T03:42:06.142698Z","iopub.execute_input":"2022-02-24T03:42:06.143528Z","iopub.status.idle":"2022-02-24T03:42:25.927191Z","shell.execute_reply.started":"2022-02-24T03:42:06.143474Z","shell.execute_reply":"2022-02-24T03:42:25.925738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Reading data\n\nfrom pandas import read_csv\n\ntrain = read_csv('/kaggle/input/airbnb-ironhack/train.csv')\ntest = read_csv('/kaggle/input/airbnb-ironhack/test.csv')\nsample = read_csv('/kaggle/input/airbnb-ironhack/sample.csv')\n\ntrain.memory_usage().sum() / 1000000 # MBs\ntest.memory_usage().sum() / 1000000 # MBs\nsample.memory_usage().sum() / 1000000 # MBs","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:25.930171Z","iopub.execute_input":"2022-02-24T03:42:25.930447Z","iopub.status.idle":"2022-02-24T03:42:41.269853Z","shell.execute_reply.started":"2022-02-24T03:42:25.930413Z","shell.execute_reply":"2022-02-24T03:42:41.268775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Less memory\n\nfrom gc import collect\nfrom numpy import dtype\n\ndef less_mem(df):\n    df = df.copy()\n    for col in df:\n        if df[col].dtype == dtype('float64'):\n            df[col] = df[col].astype('float32')\n        if df[col].dtype == dtype('int64'):\n            df[col] = df[col].astype('int32')\n    collect()\n    \n    return df\n\ntrain = less_mem(train)\ntest = less_mem(test)\nsample = less_mem(sample)\n\ntrain.memory_usage().sum() / 1000000 # MBs\ntest.memory_usage().sum() / 1000000 # MBs\nsample.memory_usage().sum() / 1000000 # MBs","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:41.273079Z","iopub.execute_input":"2022-02-24T03:42:41.273312Z","iopub.status.idle":"2022-02-24T03:42:43.114678Z","shell.execute_reply.started":"2022-02-24T03:42:41.273283Z","shell.execute_reply":"2022-02-24T03:42:43.113688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Simple EDA for y target\n\ntrain['price'].plot(kind='box')\n\n# Huge outlier, if also present in the test data, MSE will be big","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:43.117311Z","iopub.execute_input":"2022-02-24T03:42:43.117552Z","iopub.status.idle":"2022-02-24T03:42:43.388712Z","shell.execute_reply.started":"2022-02-24T03:42:43.117521Z","shell.execute_reply":"2022-02-24T03:42:43.388047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Patch y outlier\n\ntrain['y_outlier'] = 0\nidx = train['price'].argmax()\ntrain.loc[idx, 'y_outlier'] = 1\n\ntest['y_outlier'] = 0","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:43.389949Z","iopub.execute_input":"2022-02-24T03:42:43.390844Z","iopub.status.idle":"2022-02-24T03:42:43.400648Z","shell.execute_reply.started":"2022-02-24T03:42:43.390807Z","shell.execute_reply":"2022-02-24T03:42:43.399528Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Fix test_id because some string inside\n\ntest_id = test['id'].copy()\n\nidx = test_id.str.contains('\\D')==True\ntest_id.iloc[idx] = 999999\n\nidx2 = test_id.isna()\ntest_id.iloc[idx2] = 999999\n\ntest_id = test_id.astype('int32')","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:43.402941Z","iopub.execute_input":"2022-02-24T03:42:43.403454Z","iopub.status.idle":"2022-02-24T03:42:43.561093Z","shell.execute_reply.started":"2022-02-24T03:42:43.403421Z","shell.execute_reply":"2022-02-24T03:42:43.560001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Inspecting NaNs\n\ntrain.isna().sum().max()\n\ndrop_cols = train.columns[train.isna().sum() == len(train)]\ndrop_cols\n\n# There are columsn where all data is NaN!!\n\n# Also there is bathrooms train data, but not in test..., dropping","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:43.562478Z","iopub.execute_input":"2022-02-24T03:42:43.562759Z","iopub.status.idle":"2022-02-24T03:42:45.76831Z","shell.execute_reply.started":"2022-02-24T03:42:43.562725Z","shell.execute_reply":"2022-02-24T03:42:45.767635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Also, an error inside bedrooms column, there are some 't'...\n\nfrom numpy import nan\n\ntrain['bedrooms'] = train['bedrooms'].replace(to_replace='t', value=nan).astype(dtype('float32'))","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:45.769767Z","iopub.execute_input":"2022-02-24T03:42:45.770248Z","iopub.status.idle":"2022-02-24T03:42:46.093176Z","shell.execute_reply.started":"2022-02-24T03:42:45.770206Z","shell.execute_reply":"2022-02-24T03:42:46.092073Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Selecting numeric variables for this exercise\n\ntrain_ = train.select_dtypes(include='number').drop([\n   'calendar_updated', 'region_id', 'region_parent_id', 'host_id', 'requires_license', 'bathrooms'], axis=1)  #  'id', 'scrape_id', \ntrain_.shape, train_.isna().sum().sum(), train_.min().min(), train_.max().max()\n\ntest_ = test.select_dtypes(include='number').drop([\n   'calendar_updated', 'region_id', 'region_parent_id', 'host_id', 'requires_license', 'bathrooms'], axis=1)  #   'id', 'scrape_id', \ntest_.shape, test_.isna().sum().sum(), test_.min().min(), test_.max().max()\n\nfrom pandas import DataFrame, Series\nDataFrame([train_.dtypes, test_.dtypes]).T","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:46.094609Z","iopub.execute_input":"2022-02-24T03:42:46.094867Z","iopub.status.idle":"2022-02-24T03:42:46.23498Z","shell.execute_reply.started":"2022-02-24T03:42:46.094832Z","shell.execute_reply":"2022-02-24T03:42:46.234213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Simple Impute to not lose too muhc NaNs\n\nfrom sklearn.impute import SimpleImputer\nfrom pandas import DataFrame, Series\n\nsi = SimpleImputer()\n\ntrain__ = DataFrame(si.fit_transform(train_), columns=train_.columns)   #   train_.copy().dropna() \ntrain__.shape\ntest__ = DataFrame(si.fit_transform(test_), columns=test_.columns)\ntest__.shape","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:42:46.237484Z","iopub.execute_input":"2022-02-24T03:42:46.237932Z","iopub.status.idle":"2022-02-24T03:42:46.521796Z","shell.execute_reply.started":"2022-02-24T03:42:46.237871Z","shell.execute_reply":"2022-02-24T03:42:46.520704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Quick path , only numeric features\n\nfrom copy import deepcopy\n\nfrom sklearn.linear_model import LinearRegression, Lasso, Ridge, Lars\nfrom sklearn.ensemble import RandomForestRegressor, HistGradientBoostingRegressor, GradientBoostingRegressor\nfrom sklearn.tree import DecisionTreeRegressor\n\nlr, la, ri, ls = LinearRegression(), Lasso(alpha=0.01), Ridge(alpha=10), Lars(normalize=False)\ntr = DecisionTreeRegressor(min_samples_leaf=15)  # added anti-overfitting\nrf = RandomForestRegressor(min_samples_leaf=25, n_estimators=50, n_jobs=-1)\ngb = GradientBoostingRegressor(min_samples_leaf=15)\nhb = HistGradientBoostingRegressor(min_samples_leaf=50)\n\nclfs = [gb]\n\nfrom sklearn.preprocessing import RobustScaler, StandardScaler\nsi, sc = SimpleImputer(), RobustScaler()\n\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.compose import make_column_transformer, make_column_selector\n\ndata_pip = make_pipeline(StandardScaler())\ndata_pip_y = make_pipeline(StandardScaler())\n\nX, y = train__.drop('price', axis=1), train__[['price']]\nX, y = data_pip.fit_transform(X), data_pip_y.fit_transform(y)\n\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\n\nnum_splits = 4\nfolds = KFold(n_splits=num_splits, shuffle=True)\n\nests = {}\nfor clf in clfs:\n    clf\n    ests[str(clf)] = {}\n    for i, (train_idx, test_idx) in enumerate(folds.split(X)):\n        print(f'fold {i}', end=' ')\n        #if i >= 4: break\n        est = clf.fit(X[train_idx], y[train_idx])\n        score = mean_squared_error(y[test_idx], est.predict(X[test_idx]))\n        ests[str(clf)][score] = deepcopy(est)\n            \nests","metadata":{"execution":{"iopub.status.busy":"2022-02-24T04:01:08.372113Z","iopub.execute_input":"2022-02-24T04:01:08.372453Z","iopub.status.idle":"2022-02-24T04:05:54.8249Z","shell.execute_reply.started":"2022-02-24T04:01:08.372386Z","shell.execute_reply":"2022-02-24T04:05:54.8239Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Eval\n\nbest_est, best_est_score = '', 999999\nfor est in ests.keys():  # for each estimator\n    score = [v for v in ests[est]]\n    score = sum(score) / len(score)\n    print(est, score)\n    if score < best_est_score:\n        best_est_score = score\n        best_est = est\n        \nbest_est, best_est_score","metadata":{"execution":{"iopub.status.busy":"2022-02-24T04:12:20.837609Z","iopub.execute_input":"2022-02-24T04:12:20.837948Z","iopub.status.idle":"2022-02-24T04:12:20.848618Z","shell.execute_reply.started":"2022-02-24T04:12:20.837913Z","shell.execute_reply":"2022-02-24T04:12:20.84755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preds\n\nfrom numpy import column_stack, empty\n\nX = test__\n\n# TODO just take the best cv estimator, dont don mean\n\nest = ests[best_est]\nsub_preds = empty((len(X), num_splits))\nfor i, sub_est in enumerate(est.values()):\n    sub_preds[:, i] = data_pip_y.inverse_transform(sub_est.predict(data_pip.transform(X)).reshape(-1, 1)).flatten()\n    \nsub_preds = sub_preds.mean(axis=1, keepdims=True)\nsub_preds = DataFrame(sub_preds)\nsub_preds['id'] = test_id\nsub_preds","metadata":{"execution":{"iopub.status.busy":"2022-02-24T04:12:22.420014Z","iopub.execute_input":"2022-02-24T04:12:22.421024Z","iopub.status.idle":"2022-02-24T04:12:22.975809Z","shell.execute_reply.started":"2022-02-24T04:12:22.420944Z","shell.execute_reply":"2022-02-24T04:12:22.974502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Subm\n\nfrom pandas import merge\n\nsubm = sample.merge(sub_preds, left_on='id', right_on='id')\nsubm = subm.drop('price', axis=1)\nsubm.columns = ['id', 'price']\nsubm.to_csv('submission.csv', index=False)\nsubm","metadata":{"execution":{"iopub.status.busy":"2022-02-24T04:12:24.629441Z","iopub.execute_input":"2022-02-24T04:12:24.629769Z","iopub.status.idle":"2022-02-24T04:12:24.951877Z","shell.execute_reply.started":"2022-02-24T04:12:24.629735Z","shell.execute_reply":"2022-02-24T04:12:24.951166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# END\n\n# Hope you liked it!","metadata":{"execution":{"iopub.status.busy":"2022-02-24T03:52:33.282556Z","iopub.execute_input":"2022-02-24T03:52:33.283525Z","iopub.status.idle":"2022-02-24T03:52:33.288022Z","shell.execute_reply.started":"2022-02-24T03:52:33.283471Z","shell.execute_reply":"2022-02-24T03:52:33.287104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}