{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Spaceship Titanic","metadata":{}},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import os, warnings\nimport numpy as np, pandas as pd\nfrom xgboost import XGBClassifier\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.preprocessing import StandardScaler, MinMaxScaler\nfrom hyperopt import hp, fmin, tpe, space_eval, Trials, STATUS_OK\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-05-27T06:26:49.768983Z","iopub.execute_input":"2022-05-27T06:26:49.769364Z","iopub.status.idle":"2022-05-27T06:26:51.689997Z","shell.execute_reply.started":"2022-05-27T06:26:49.769269Z","shell.execute_reply":"2022-05-27T06:26:51.689285Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Parameters","metadata":{}},{"cell_type":"code","source":"C_Random_Seed = 22\nos.environ[\"HYPEROPT_FMIN_SEED\"] = f'{C_Random_Seed}'\nwarnings.filterwarnings('ignore')\n\nC_Selected_Model = 'xgb'\nC_Debug = False\n\nC_Tunable_Params = {\n    'rnf': {'max_depth': hp.choice('rnf.max_depth', np.arange(2, 5, dtype = int)),\n            'n_estimators': hp.choice('rnf.n_estimators', np.arange(50, 400, dtype = int)),          \n           },\n\n    'xgb': {'max_depth': hp.choice('xgb.max_depth', np.arange(2, 5, dtype = int)),\n            'learning_rate': hp.quniform('xgb.learning_rate', 0.01, 0.05, 0.01),\n            'n_estimators': hp.choice('xgb.n_estimators', np.arange(50, 400, dtype = int)),\n            'subsample': hp.quniform('xgb.subsample', 0.1, 1.0, 0.1),\n            'gamma': hp.quniform('xgb.gamma', 0.0, 0.5, 0.1),\n            'min_child_weight': hp.quniform('xgb.min_child_weight', 1, 10, 1),          \n           },\n    \n    'gbc': {'loss': hp.choice('gbc.loss', ['log_loss', 'exponential']),\n            'max_depth': hp.choice('gbc.max_depth', np.arange(2, 5, dtype = int)),\n            'learning_rate': hp.quniform('gbc.learning_rate', 0.05, 0.4, 0.01),\n            'n_estimators': hp.choice('gbc.n_estimators', np.arange(50, 400, dtype = int)),\n            'subsample': hp.quniform('gbc.subsample', 0.1, 1.0, 0.1),\n            'criterion': hp.choice('gbc.criterion', ['friedman_mse', 'mse', 'squared_error']),\n            },\n\n    'svc': {'C': hp.quniform('svc.C', 0.1, 1.0, 0.1),\n            'kernel': hp.choice('svc.kernel', ['linear', 'poly', 'rbf', 'sigmoid', 'precomputed']),\n            'degree': hp.choice('svc.degree', np.arange(2, 4, dtype = int)),\n            'gamma': hp.choice('svc.gamma', ['scale', 'auto']),\n            },    \n}            ","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:26:51.691997Z","iopub.execute_input":"2022-05-27T06:26:51.692689Z","iopub.status.idle":"2022-05-27T06:26:51.71761Z","shell.execute_reply.started":"2022-05-27T06:26:51.692641Z","shell.execute_reply":"2022-05-27T06:26:51.716767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Preprocessing","metadata":{}},{"cell_type":"code","source":"def get_data():\n    \"\"\"\n    1. Getting the data from the input path\n    2. Split the training and test data into features and targets\n    \"\"\"\n    train_df = pd.read_csv('../input/spaceship-titanic/train.csv')\n    test_df = pd.read_csv('../input/spaceship-titanic/test.csv')\n    X_train = train_df.drop(['Transported'], axis = 1)\n    y_train = train_df['Transported']\n    X_test = test_df.copy()\n    return X_train, y_train, X_test\n\n\ndef preprocess(df):\n    \"\"\"\n    1. Handle missing values\n      a. Cabin: Fill a dummy value in the given format: Deck/Num/Side\n      b. Age: Assume people whose age is missing as adults\n      c. Side: Assume missing values are Port\n      d. CryoSleep: People will need to spend money (key - FoodCourt), if they are not on CryoSleep.\n      \n    2. Encode categorical variables\n      a. Ordinal Encoding:\n          - Deck : Since people at lower decks have a lesser chance of escaping. ABCDEFGT Bottom-Up.\n      b. One Hot Encoding:\n          - Side (Port / Starboard)\n          - Age (Child / Adult)\n          - CryoSleep (True / False)\n          - VIP (True / False)\n          - HomePlanet\n          - Destination\n          \n    3. Feature Engineering\n      a. Regular, Luxury and Total Spends.\n      b. Remove columns that do not provide any useful information.\n    \"\"\"\n    \n    df['Age'].fillna(19, inplace = True)\n    df['Age'] = df['Age'].apply(lambda x: 1 if x <= 18 else 0)\n    df['Name'].fillna('Noname', inplace = True)\n    \n    df['Cabin'].fillna('0/0/0', inplace = True)\n    df['Deck'] = df['Cabin'].apply(lambda x: str(x).split('/')[0])\n    df['Deck'] = df['Deck'].apply(lambda x: '0ABCDEFGT'.index(x))\n    df['Side'] = df['Cabin'].apply(lambda x: str(x).split('/')[2])\n    df['Side'] = df['Side'].replace({'0': 'P'})\n    \n    money_cols = ['RoomService', 'FoodCourt', 'ShoppingMall', 'Spa', 'VRDeck']\n    for col in money_cols:\n        df[col] = df[col].fillna(0)\n    df['Regular'] =  df[['FoodCourt', 'ShoppingMall']].sum(axis = 1)  \n    df['Luxury'] =  df[['RoomService', 'Spa', 'VRDeck']].sum(axis = 1)  \n    df['Total_Spent'] = df[money_cols].sum(axis = 1)\n    \n    df.loc[(df.CryoSleep.isnull()) & (df.Total_Spent == 0), 'CryoSleep'] = True\n    df.loc[(df.CryoSleep.isnull()) & (df.Total_Spent != 0), 'CryoSleep'] = False\n    \n    df['Id'] = df.PassengerId.str[:4]\n    df['Group'] = df.Id.duplicated(keep = False).astype(int)\n    \n    df['Name'] = df['Name'].apply(lambda x: x.split()[-1])\n    df['Relatives'] = df.Name.duplicated(keep = False).astype(int)\n    df.loc[df.Name == 'Noname', 'Relatives'] = 0\n    \n    df.drop(money_cols + ['Name', 'Cabin', 'PassengerId', 'Id'], axis = 1, inplace = True)\n    df = pd.get_dummies(df, columns = ['HomePlanet', 'CryoSleep', 'Destination', 'VIP', 'Side'], drop_first = True)\n    return df\n\n\ndef feature_transformations(train, test):\n    \"\"\"\n    1. Feature Transformation of continuous features.\n      a. Deck is distributed across a narrow range. MinMax Scaling would be suitable.\n      b. Expenditures are distributed across a wide range. Log Transformations would be ideal.\n    2. Necessary if the algorithm used is not tree based.\n    \"\"\"\n              \n    for col in ['Total_Spent', 'Regular', 'Luxury']:\n        train[col] = np.log1p(train[col])\n        test[col] = np.log1p(test[col])\n        \n    for col_ in ['Deck']:\n        sc_X = MinMaxScaler(feature_range = (0, 1))\n        train.loc[:, col_] = sc_X.fit_transform(train.loc[:, col_].values.reshape(-1, 1))\n        test.loc[:, col_] = sc_X.transform(test.loc[:, col_].values.reshape(-1, 1))         \n    return train, test","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:26:51.719405Z","iopub.execute_input":"2022-05-27T06:26:51.71999Z","iopub.status.idle":"2022-05-27T06:26:51.745536Z","shell.execute_reply.started":"2022-05-27T06:26:51.719945Z","shell.execute_reply":"2022-05-27T06:26:51.744869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Modelling Approach","metadata":{}},{"cell_type":"code","source":"def get_model_instance(mod_type_, params):\n    \"\"\"\n    Create a model instance with the provided parameters.\n    \"\"\"\n    if mod_type_ == 'rnf':\n        selected_model = RandomForestClassifier(**params, random_state = C_Random_Seed)\n    elif mod_type_ == 'xgb':\n        selected_model = XGBClassifier(**params, random_state = C_Random_Seed)\n    elif mod_type_ == 'gbc':\n        selected_model = GradientBoostingClassifier(**params, random_state = C_Random_Seed)           \n    return selected_model\n   \n\ndef fine_tune_model(X_train, y_train, mod_type_):\n    \"\"\"\n    Tune the hyperparameters for the model selected.\n    \"\"\"\n    def objective(params):\n        model = get_model_instance(mod_type_, params)\n        loss_metric = -1 * cross_val_score(model, X_train, y_train, cv = 10, scoring = 'roc_auc')\n        return {'loss': np.mean(loss_metric), 'loss_on_folds': loss_metric, 'status': STATUS_OK}\n\n    fmin_trials = Trials()\n    search_space = hp.choice('model_type', [C_Tunable_Params[mod_type_]])\n    best_params = fmin(fn = objective, space = search_space, algo = tpe.suggest, trials = fmin_trials, max_evals = 100, \n                       show_progressbar = False, verbose = False, rstate = np.random.default_rng(C_Random_Seed))    \n    best = fmin_trials.best_trial['result']\n    best['params'] = space_eval(search_space, best_params)\n    best['type'] = mod_type_\n    return best    \n    \n    \ndef model_selection_01(X_train, y_train, X_test):\n    \"\"\"\n    1. Select the appropriate model and tune the hyperparameters\n    2. Return feature importances and predictions\n    \n    \"\"\"\n    if C_Selected_Model:\n        model_pool = [C_Selected_Model]\n    else:\n        model_pool = list(C_Tunable_Params.keys())        \n                       \n    model_summary_list = []    \n    for mod_type in model_pool:\n        best = fine_tune_model(X_train, y_train, mod_type)\n        model_summary_list.append(best)\n    \n    model_summary_df = pd.DataFrame(model_summary_list)\n    best_model_summary = model_summary_df.iloc[model_summary_df.loss.argmin()]\n    best_model = get_model_instance(best_model_summary['type'], best_model_summary['params'])\n\n    best_model.fit(X_train, y_train)\n    predictions = best_model.predict(X_test)\n    feat_imp = list(zip(best_model.feature_names_in_, best_model.feature_importances_))\n    feat_imp = sorted(feat_imp, key = lambda x: x[1], reverse = True)\n    \n    return predictions, feat_imp","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:26:51.748076Z","iopub.execute_input":"2022-05-27T06:26:51.748666Z","iopub.status.idle":"2022-05-27T06:26:51.767393Z","shell.execute_reply.started":"2022-05-27T06:26:51.748617Z","shell.execute_reply":"2022-05-27T06:26:51.766661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def finalize(y_pred):\n    \"\"\"\n    1. Get the template from the sample submission file.\n    2. Add predictions to the template.\n    3. Export to submission.csv\n    \"\"\"\n    submission = pd.read_csv('../input/spaceship-titanic/sample_submission.csv')\n    submission['Transported'] = y_pred\n    submission['Transported'] = submission['Transported'].replace({1: 'True', 0: 'False'})\n    submission.to_csv('./submission.csv', index = False)","metadata":{"_kg_hide-input":false,"execution":{"iopub.status.busy":"2022-05-27T06:26:51.771044Z","iopub.execute_input":"2022-05-27T06:26:51.771711Z","iopub.status.idle":"2022-05-27T06:26:51.783891Z","shell.execute_reply.started":"2022-05-27T06:26:51.771658Z","shell.execute_reply":"2022-05-27T06:26:51.78286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Driver","metadata":{}},{"cell_type":"code","source":"print('Getting the data..')\nX_train, y_train, X_test = get_data()\n\nprint('Preprocessing..')\nX_train = preprocess(X_train)\nX_test = preprocess(X_test)\nX_train, X_test = feature_transformations(X_train, X_test)\n\nif C_Debug:\n    pass\nelse:\n    print('Model training..')\n    y_pred, feature_importances = model_selection_01(X_train, y_train, X_test)\n    print('Finalizing..')\n    finalize(y_pred)","metadata":{"execution":{"iopub.status.busy":"2022-05-27T06:26:51.785792Z","iopub.execute_input":"2022-05-27T06:26:51.786475Z","iopub.status.idle":"2022-05-27T06:26:52.053603Z","shell.execute_reply.started":"2022-05-27T06:26:51.786417Z","shell.execute_reply":"2022-05-27T06:26:52.052668Z"},"trusted":true},"execution_count":null,"outputs":[]}]}