{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tau to Muon Decay - Finding New Physics","metadata":{}},{"cell_type":"markdown","source":"> ### Description : The aim of this notebook is to find a phenomenon that is not already known to exist – charged lepton flavour violation – thereby helping to establish \"new physics\". ","metadata":{}},{"cell_type":"markdown","source":"# Problem Statement: From a list of collision events and their properties, predict whether a τ → 3μ decay happened in this collision. This τ → 3μ is currently assumed by scientists not to happen, and the goal of this competition is to discover τ → 3μ happening more frequently than scientists currently can understand.","metadata":{}},{"cell_type":"markdown","source":"## Data : https://www.kaggle.com/competitions/flavours-of-physics-kernels-only/data","metadata":{}},{"cell_type":"markdown","source":"# Imports and File Reads","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-10T12:37:03.931054Z","iopub.execute_input":"2022-09-10T12:37:03.931568Z","iopub.status.idle":"2022-09-10T12:37:03.941515Z","shell.execute_reply.started":"2022-09-10T12:37:03.931522Z","shell.execute_reply":"2022-09-10T12:37:03.940311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nfrom sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\nfrom IPython.display import display\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport math\nimport dateutil.parser\nfrom sklearn.preprocessing import MinMaxScaler\nimport numpy as np\nfrom sklearn.ensemble import GradientBoostingClassifier\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.tree import DecisionTreeRegressor\nimport sklearn.metrics as metrics\nfrom sklearn.metrics import accuracy_score,confusion_matrix, f1_score, confusion_matrix, f1_score, classification_report, roc_curve, roc_auc_score, auc, precision_recall_curve, precision_score, recall_score","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-09-10T12:37:13.245438Z","iopub.execute_input":"2022-09-10T12:37:13.245800Z","iopub.status.idle":"2022-09-10T12:37:14.367813Z","shell.execute_reply.started":"2022-09-10T12:37:13.245748Z","shell.execute_reply":"2022-09-10T12:37:14.366844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import xgboost as xgb","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2022-09-10T12:37:15.349531Z","iopub.execute_input":"2022-09-10T12:37:15.349918Z","iopub.status.idle":"2022-09-10T12:37:16.355770Z","shell.execute_reply.started":"2022-09-10T12:37:15.349861Z","shell.execute_reply":"2022-09-10T12:37:16.354858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.set_printoptions(linewidth=140)\npd.set_option('display.width', 140)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:16.359661Z","iopub.execute_input":"2022-09-10T12:37:16.360233Z","iopub.status.idle":"2022-09-10T12:37:16.365542Z","shell.execute_reply.started":"2022-09-10T12:37:16.360173Z","shell.execute_reply":"2022-09-10T12:37:16.364572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Read in the data","metadata":{}},{"cell_type":"code","source":"path = '/kaggle/input/flavours-of-physics-kernels-only/'\n\ntrain = pd.read_csv(path+'training.csv.zip', index_col='id')\ntest  = pd.read_csv(path+'test.csv.zip', index_col = \"id\")","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:20.265196Z","iopub.execute_input":"2022-09-10T12:37:20.265844Z","iopub.status.idle":"2022-09-10T12:37:36.020940Z","shell.execute_reply.started":"2022-09-10T12:37:20.265775Z","shell.execute_reply":"2022-09-10T12:37:36.020001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(3)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:44.702916Z","iopub.execute_input":"2022-09-10T12:37:44.703303Z","iopub.status.idle":"2022-09-10T12:37:44.738829Z","shell.execute_reply.started":"2022-09-10T12:37:44.703240Z","shell.execute_reply":"2022-09-10T12:37:44.737958Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, test.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:45.086513Z","iopub.execute_input":"2022-09-10T12:37:45.087137Z","iopub.status.idle":"2022-09-10T12:37:45.093816Z","shell.execute_reply.started":"2022-09-10T12:37:45.087068Z","shell.execute_reply":"2022-09-10T12:37:45.093133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Nothing is null, that's good","metadata":{}},{"cell_type":"code","source":"train.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:49.687682Z","iopub.execute_input":"2022-09-10T12:37:49.688308Z","iopub.status.idle":"2022-09-10T12:37:49.707300Z","shell.execute_reply.started":"2022-09-10T12:37:49.688261Z","shell.execute_reply":"2022-09-10T12:37:49.706097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### There are a few variables such as SPDhits and p0IPSig that have larger numerical values than the others.","metadata":{}},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:53.504149Z","iopub.execute_input":"2022-09-10T12:37:53.504586Z","iopub.status.idle":"2022-09-10T12:37:53.773663Z","shell.execute_reply.started":"2022-09-10T12:37:53.504516Z","shell.execute_reply":"2022-09-10T12:37:53.772426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 'production', 'mass', 'min_ANNmuon' aren't in the test, so they have to be removed\n","metadata":{}},{"cell_type":"code","source":"\nall_num_cols = ['LifeTime', 'dira', 'FlightDistance', 'FlightDistanceError', 'IP', 'IPSig', 'VertexChi2', 'pt', 'DOCAone', 'DOCAtwo', 'DOCAthree',\n       'IP_p0p2', 'IP_p1p2', 'isolationa', 'isolationb', 'isolationc', 'isolationd', 'isolatione', 'isolationf', 'iso', 'CDF1', 'CDF2',\n       'CDF3', 'ISO_SumBDT', 'p0_IsoBDT', 'p1_IsoBDT', 'p2_IsoBDT', 'p0_track_Chi2Dof', 'p1_track_Chi2Dof', 'p2_track_Chi2Dof', 'p0_IP',\n       'p1_IP', 'p2_IP', 'p0_IPSig', 'p1_IPSig', 'p2_IPSig', 'p0_pt', 'p1_pt', 'p2_pt', 'p0_p', 'p1_p', 'p2_p', 'p0_eta', 'p1_eta',\n       'p2_eta', 'SPDhits']","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:56.511967Z","iopub.execute_input":"2022-09-10T12:37:56.512358Z","iopub.status.idle":"2022-09-10T12:37:56.518901Z","shell.execute_reply.started":"2022-09-10T12:37:56.512291Z","shell.execute_reply":"2022-09-10T12:37:56.517771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y = train[\"signal\"]\nx = train[all_num_cols] ","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:57.742138Z","iopub.execute_input":"2022-09-10T12:37:57.742547Z","iopub.status.idle":"2022-09-10T12:37:57.754469Z","shell.execute_reply.started":"2022-09-10T12:37:57.742483Z","shell.execute_reply":"2022-09-10T12:37:57.753473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Scale the data, so everyone is speaking the same numerical precision","metadata":{}},{"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\nsc = StandardScaler()\nX_scaled= pd.DataFrame(sc.fit_transform(x), columns=all_num_cols)\nX_scaled.var()","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:37:59.044117Z","iopub.execute_input":"2022-09-10T12:37:59.044681Z","iopub.status.idle":"2022-09-10T12:37:59.133210Z","shell.execute_reply.started":"2022-09-10T12:37:59.044617Z","shell.execute_reply":"2022-09-10T12:37:59.132194Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_scaled.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:38:03.292747Z","iopub.execute_input":"2022-09-10T12:38:03.293427Z","iopub.status.idle":"2022-09-10T12:38:03.321095Z","shell.execute_reply.started":"2022-09-10T12:38:03.293374Z","shell.execute_reply":"2022-09-10T12:38:03.320161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Train Validation Split","metadata":{}},{"cell_type":"markdown","source":"#### The test data that's supplied doesn't contain a signal column, so it can't be used for testing the robustness of the model. It will be used only for making the predictions at the end. So, split the train into train and validation sets and the latter would be used as the \"test\" set.","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_train, X_val, y_train, y_val = train_test_split(X_scaled, y, test_size=0.25, random_state=42)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:38:17.419721Z","iopub.execute_input":"2022-09-10T12:38:17.420117Z","iopub.status.idle":"2022-09-10T12:38:17.450093Z","shell.execute_reply.started":"2022-09-10T12:38:17.420045Z","shell.execute_reply":"2022-09-10T12:38:17.449159Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.shape, x.shape, X_train.shape, X_val.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:38:20.819113Z","iopub.execute_input":"2022-09-10T12:38:20.819531Z","iopub.status.idle":"2022-09-10T12:38:20.826177Z","shell.execute_reply.started":"2022-09-10T12:38:20.819468Z","shell.execute_reply":"2022-09-10T12:38:20.825126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Fit a model without any adjustments to check what variables are important","metadata":{}},{"cell_type":"code","source":"print(\"Random Forest\")\nrf = RandomForestClassifier(n_estimators=100, random_state=11)\nrf.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:38:22.043311Z","iopub.execute_input":"2022-09-10T12:38:22.044009Z","iopub.status.idle":"2022-09-10T12:39:03.546840Z","shell.execute_reply.started":"2022-09-10T12:38:22.043945Z","shell.execute_reply":"2022-09-10T12:39:03.545659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def rf_feat_importance(m, df):\n    return pd.DataFrame({'cols':df.columns, 'imp':m.feature_importances_}\n                       ).sort_values('imp', ascending=False)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.549425Z","iopub.execute_input":"2022-09-10T12:39:03.550001Z","iopub.status.idle":"2022-09-10T12:39:03.555746Z","shell.execute_reply.started":"2022-09-10T12:39:03.549793Z","shell.execute_reply":"2022-09-10T12:39:03.554570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fi = rf_feat_importance(rf, X_train)\nfi.sort_values(\"imp\", ascending = False)[:10]","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.557231Z","iopub.execute_input":"2022-09-10T12:39:03.557576Z","iopub.status.idle":"2022-09-10T12:39:03.598315Z","shell.execute_reply.started":"2022-09-10T12:39:03.557525Z","shell.execute_reply":"2022-09-10T12:39:03.597478Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_fi(fi):\n    return fi.plot('cols', 'imp', 'barh', figsize=(12,7), legend=False)\n\nplot_fi(fi[:20]);\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.599629Z","iopub.execute_input":"2022-09-10T12:39:03.599936Z","iopub.status.idle":"2022-09-10T12:39:03.884403Z","shell.execute_reply.started":"2022-09-10T12:39:03.599885Z","shell.execute_reply":"2022-09-10T12:39:03.883452Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Use the selected columns to create a slice of the training data","metadata":{}},{"cell_type":"code","source":"selected_cols = [\"IPSig\", \"IP\", \"dira\", \"SPDhits\", \"VertexChi2\", \"p0_track_Chi2Dof\", \"ISO_SumBDT\", \"iso\", \"LifeTime\", \"p0_IsoBDT\", \"p0_IPSig\", \"isolatione\" ]","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.887015Z","iopub.execute_input":"2022-09-10T12:39:03.887310Z","iopub.status.idle":"2022-09-10T12:39:03.892254Z","shell.execute_reply.started":"2022-09-10T12:39:03.887264Z","shell.execute_reply":"2022-09-10T12:39:03.891181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train = X_train[selected_cols] ","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.893754Z","iopub.execute_input":"2022-09-10T12:39:03.894127Z","iopub.status.idle":"2022-09-10T12:39:03.905345Z","shell.execute_reply.started":"2022-09-10T12:39:03.894078Z","shell.execute_reply":"2022-09-10T12:39:03.904479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.906707Z","iopub.execute_input":"2022-09-10T12:39:03.907053Z","iopub.status.idle":"2022-09-10T12:39:03.918667Z","shell.execute_reply.started":"2022-09-10T12:39:03.907011Z","shell.execute_reply":"2022-09-10T12:39:03.917825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Not ideal profiles but that's expected in the experiments. Hopefully, Random Forests and Boosting algos will take care of them","metadata":{}},{"cell_type":"code","source":"X_train.hist(bins=30, figsize=(20, 14))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:03.920041Z","iopub.execute_input":"2022-09-10T12:39:03.920653Z","iopub.status.idle":"2022-09-10T12:39:06.298015Z","shell.execute_reply.started":"2022-09-10T12:39:03.920590Z","shell.execute_reply":"2022-09-10T12:39:06.297152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Good that data isn't just only signal or only background. 3:2 ratio seems ok!","metadata":{}},{"cell_type":"code","source":"y_train.hist()","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:06.299300Z","iopub.execute_input":"2022-09-10T12:39:06.299660Z","iopub.status.idle":"2022-09-10T12:39:06.472720Z","shell.execute_reply.started":"2022-09-10T12:39:06.299602Z","shell.execute_reply":"2022-09-10T12:39:06.471767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"####  Don't run these, they actually made things worse","metadata":{}},{"cell_type":"code","source":"# transformations\nX_train['SPDhits'] = np.log(X_train['SPDhits'])\nX_train['p0_IPSig'] = np.log(X_train['p0_IPSig'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[\"SPDhits\"].hist(bins=30, figsize=(5, 3))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_train[\"p0_IPSig\"].hist(bins=30, figsize=(5, 3))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Don't run ends","metadata":{}},{"cell_type":"markdown","source":"# Correlation Matrix for the variables selected. Some strong correlations owe their values to the method of production","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(15,10))\nsns.heatmap(X_train.corr(), annot=True, cbar=False, fmt='.1f', cmap='summer_r')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:14.131968Z","iopub.execute_input":"2022-09-10T12:39:14.132329Z","iopub.status.idle":"2022-09-10T12:39:14.950164Z","shell.execute_reply.started":"2022-09-10T12:39:14.132281Z","shell.execute_reply":"2022-09-10T12:39:14.949104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Functions","metadata":{}},{"cell_type":"markdown","source":"### Root mean squared error","metadata":{}},{"cell_type":"code","source":"def r_mse(pred,y):\n    return round(math.sqrt(((pred-y)**2).mean()), 6)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:25:11.882901Z","iopub.execute_input":"2022-09-10T14:25:11.883306Z","iopub.status.idle":"2022-09-10T14:25:11.888472Z","shell.execute_reply.started":"2022-09-10T14:25:11.883253Z","shell.execute_reply":"2022-09-10T14:25:11.887635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion matrix","metadata":{}},{"cell_type":"code","source":"def con_matrix(y_val, val_preds_bin):\n    matrix = confusion_matrix(y_val, val_preds_bin)\n    matrix = matrix.astype('float') / matrix.sum(axis=1)[:, np.newaxis]\n\n    # Build the plot\n    plt.figure(figsize=(8,5))\n    sns.set(font_scale=1.4)\n    sns.heatmap(matrix, annot=True, annot_kws={'size':10},\n                cmap=plt.cm.Greens, linewidths=0.2)\n\n    # Add labels to the plot\n    class_names = ['Signal', 'Background']\n    tick_marks = np.arange(len(class_names))\n    tick_marks2 = tick_marks + 0.5\n    plt.xticks(tick_marks, class_names, rotation=0)\n    plt.yticks(tick_marks2, class_names, rotation=0)\n    plt.xlabel('Predicted label')\n    plt.ylabel('True label')\n    plt.title('Confusion Matrix for Ensemble Model')\n    plt.show()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:45:07.933396Z","iopub.execute_input":"2022-09-10T14:45:07.933790Z","iopub.status.idle":"2022-09-10T14:45:07.942752Z","shell.execute_reply.started":"2022-09-10T14:45:07.933739Z","shell.execute_reply":"2022-09-10T14:45:07.941967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Metrics","metadata":{}},{"cell_type":"code","source":"def metric_report(y_val, pred_val):\n    print(\"Accuracy Random_Forest:\", accuracy_score(y_val, pred_val))\n    print(\"Precision Random_Forest:\", precision_score(y_val, pred_val))\n    print(\"Recall Random_Forest:\", recall_score(y_val, pred_val))\n    print(\"F1 Score Random_Forest:\", f1_score(y_val, pred_val))\n\n    ","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:49:23.287085Z","iopub.execute_input":"2022-09-10T14:49:23.287489Z","iopub.status.idle":"2022-09-10T14:49:23.293603Z","shell.execute_reply.started":"2022-09-10T14:49:23.287423Z","shell.execute_reply":"2022-09-10T14:49:23.292735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### ROC Curve","metadata":{}},{"cell_type":"code","source":"def roc_curvePlot(y_val, val_preds):\n    \n    fpr, tpr, threshold = metrics.roc_curve(y_val, val_preds)\n    roc_auc = metrics.auc(fpr, tpr)\n\n    import matplotlib.pyplot as plt\n    plt.style.use('default')\n    plt.title('ROC curve')\n    plt.plot(tpr, 1-fpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n    plt.legend(loc = 'lower right')\n    plt.plot([0, 1], [1, 0],'r--')\n    plt.xlim([0, 1])\n    plt.ylim([0, 1])\n    plt.xlabel('Signal efficiency (TPR)')\n    plt.ylabel('Background rejection (1-FPR)')\n    plt.show()\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Models","metadata":{}},{"cell_type":"markdown","source":"## Decision Tree","metadata":{}},{"cell_type":"code","source":"m = DecisionTreeRegressor()\nm.fit(X_train, y_train);","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:46.053204Z","iopub.execute_input":"2022-09-10T12:39:46.053708Z","iopub.status.idle":"2022-09-10T12:39:47.068517Z","shell.execute_reply.started":"2022-09-10T12:39:46.053639Z","shell.execute_reply":"2022-09-10T12:39:47.067577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X_val = X_val[selected_cols]\nX_val.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:56.656435Z","iopub.execute_input":"2022-09-10T12:39:56.656826Z","iopub.status.idle":"2022-09-10T12:39:56.665122Z","shell.execute_reply.started":"2022-09-10T12:39:56.656771Z","shell.execute_reply":"2022-09-10T12:39:56.664317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_val = m.predict(X_val)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:39:58.722760Z","iopub.execute_input":"2022-09-10T12:39:58.723440Z","iopub.status.idle":"2022-09-10T12:39:58.734850Z","shell.execute_reply.started":"2022-09-10T12:39:58.723388Z","shell.execute_reply":"2022-09-10T12:39:58.733689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_val.shape, pred_val.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:40:00.714652Z","iopub.execute_input":"2022-09-10T12:40:00.715049Z","iopub.status.idle":"2022-09-10T12:40:00.720197Z","shell.execute_reply.started":"2022-09-10T12:40:00.714988Z","shell.execute_reply":"2022-09-10T12:40:00.719545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r_mse(pred_val, y_val)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:40:01.273954Z","iopub.execute_input":"2022-09-10T12:40:01.274500Z","iopub.status.idle":"2022-09-10T12:40:01.280128Z","shell.execute_reply.started":"2022-09-10T12:40:01.274428Z","shell.execute_reply":"2022-09-10T12:40:01.279470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Random Forests","metadata":{}},{"cell_type":"code","source":"# A rf classifier again\nprint(\"Random Forest\")\ntrees = [10, 50, 100, 150, 200,250, 300,350, 400,450, 500, 550, 600, 650]\nerr = []\ntm = []\nfor t in trees:\n    start = time.time()\n    print(\"Preparing trees: \", t)\n    rf = RandomForestClassifier(n_estimators=t, random_state=11)\n    rf.fit(X_train, y_train)\n    pred_val_rf = rf.predict(X_val)\n    err.append(r_mse(pred_val_rf, y_val))\n    end = time.time()\n    tm.append(end-start)\n    print(f\" For Tree with depth {i}, it took {end - start} second\")","metadata":{"execution":{"iopub.status.busy":"2022-09-10T12:47:04.955720Z","iopub.execute_input":"2022-09-10T12:47:04.956065Z","iopub.status.idle":"2022-09-10T13:01:17.793229Z","shell.execute_reply.started":"2022-09-10T12:47:04.956026Z","shell.execute_reply":"2022-09-10T13:01:17.791996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Adding trees after a while doesn't help!","metadata":{}},{"cell_type":"code","source":"plt.plot(trees, err)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:09:03.399185Z","iopub.execute_input":"2022-09-10T13:09:03.399705Z","iopub.status.idle":"2022-09-10T13:09:03.548614Z","shell.execute_reply.started":"2022-09-10T13:09:03.399659Z","shell.execute_reply":"2022-09-10T13:09:03.547595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Time increases linearly, I was expecting exponential","metadata":{}},{"cell_type":"code","source":"plt.plot(trees, tm)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:09:21.885082Z","iopub.execute_input":"2022-09-10T13:09:21.885698Z","iopub.status.idle":"2022-09-10T13:09:22.040872Z","shell.execute_reply.started":"2022-09-10T13:09:21.885427Z","shell.execute_reply":"2022-09-10T13:09:22.039761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.plot([r_mse(pred_val[:i+1].mean(0), y_val) for i in range(200)]);\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:10:20.048182Z","iopub.execute_input":"2022-09-10T13:10:20.048579Z","iopub.status.idle":"2022-09-10T13:10:20.296525Z","shell.execute_reply.started":"2022-09-10T13:10:20.048523Z","shell.execute_reply":"2022-09-10T13:10:20.295398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Decent scores","metadata":{}},{"cell_type":"code","source":"metric_report(y_val, pred_val_rf)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:48:19.506315Z","iopub.execute_input":"2022-09-10T14:48:19.506710Z","iopub.status.idle":"2022-09-10T14:48:19.538117Z","shell.execute_reply.started":"2022-09-10T14:48:19.506660Z","shell.execute_reply":"2022-09-10T14:48:19.536976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(y_val, pred_val_rf)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:10:54.992480Z","iopub.execute_input":"2022-09-10T13:10:54.992894Z","iopub.status.idle":"2022-09-10T13:10:55.018729Z","shell.execute_reply.started":"2022-09-10T13:10:54.992829Z","shell.execute_reply":"2022-09-10T13:10:55.017524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Confusion matrix","metadata":{}},{"cell_type":"code","source":"con_matrix(y_val, pred_val_rf)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:45:44.412777Z","iopub.execute_input":"2022-09-10T14:45:44.413154Z","iopub.status.idle":"2022-09-10T14:45:45.099458Z","shell.execute_reply.started":"2022-09-10T14:45:44.413103Z","shell.execute_reply":"2022-09-10T14:45:45.098448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Xgboost","metadata":{}},{"cell_type":"code","source":"print(\"XGBoost\")\nparams = {\"objective\": \"binary:logistic\",\n          \"base_Score\": 0.5,\n          \"eta\": 0.3,\n          \"max_depth\": 5,\n          \"min_child_weight\": 3,\n          \"silent\": 1,\n          \"subsample\": 0.7,\n          \"colsample_bytree\": 0.7,\n          \"seed\": 1}\nnum_trees=200","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:12:31.758562Z","iopub.execute_input":"2022-09-10T13:12:31.759206Z","iopub.status.idle":"2022-09-10T13:12:31.766233Z","shell.execute_reply.started":"2022-09-10T13:12:31.759153Z","shell.execute_reply":"2022-09-10T13:12:31.764909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gbm = xgb.train(params, xgb.DMatrix(X_train, y_train), num_trees)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:12:33.924353Z","iopub.execute_input":"2022-09-10T13:12:33.924892Z","iopub.status.idle":"2022-09-10T13:12:39.817555Z","shell.execute_reply.started":"2022-09-10T13:12:33.924845Z","shell.execute_reply":"2022-09-10T13:12:39.816533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_gbm = gbm.predict(xgb.DMatrix(X_val))","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:12:39.819434Z","iopub.execute_input":"2022-09-10T13:12:39.820211Z","iopub.status.idle":"2022-09-10T13:12:39.902489Z","shell.execute_reply.started":"2022-09-10T13:12:39.820151Z","shell.execute_reply":"2022-09-10T13:12:39.901511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"r_mse(pred_gbm, y_val)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:12:42.897557Z","iopub.execute_input":"2022-09-10T13:12:42.899151Z","iopub.status.idle":"2022-09-10T13:12:42.905686Z","shell.execute_reply.started":"2022-09-10T13:12:42.899080Z","shell.execute_reply":"2022-09-10T13:12:42.904395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Xgboost doesn't push the values to floor or ceiling, so we'll have to do it explicitly for the accuracy score and other metrics","metadata":{}},{"cell_type":"code","source":"pred_gbm","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:12:45.414738Z","iopub.execute_input":"2022-09-10T13:12:45.415102Z","iopub.status.idle":"2022-09-10T13:12:45.421588Z","shell.execute_reply.started":"2022-09-10T13:12:45.415050Z","shell.execute_reply":"2022-09-10T13:12:45.420833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred_gbm_bin =[]\nfor val in pred_gbm:\n    if val > 0.5:\n        pred_gbm_bin.append(1)\n    else:\n        pred_gbm_bin.append(0)\n   \n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:13:01.878351Z","iopub.execute_input":"2022-09-10T13:13:01.879045Z","iopub.status.idle":"2022-09-10T13:13:01.918276Z","shell.execute_reply.started":"2022-09-10T13:13:01.878976Z","shell.execute_reply":"2022-09-10T13:13:01.917477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric_report(y_val, pred_gbm_bin)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:49:37.725158Z","iopub.execute_input":"2022-09-10T14:49:37.725771Z","iopub.status.idle":"2022-09-10T14:49:37.785183Z","shell.execute_reply.started":"2022-09-10T14:49:37.725692Z","shell.execute_reply":"2022-09-10T14:49:37.784109Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"confusion_matrix(y_val, pred_gbm_bin)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:13:06.074275Z","iopub.execute_input":"2022-09-10T13:13:06.074818Z","iopub.status.idle":"2022-09-10T13:13:06.103404Z","shell.execute_reply.started":"2022-09-10T13:13:06.074758Z","shell.execute_reply":"2022-09-10T13:13:06.102309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## These models are not performing amazingly well! MSE is quite high but still classification seems not too bad. Out of ~16k, 14k seem to be correctly classified.","metadata":{}},{"cell_type":"markdown","source":"## Just for the sake of it, what does old Logistic Regression says?","metadata":{}},{"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nclf = LogisticRegression(random_state=0).fit(X_train, y_train)\npred_lr = clf.predict(X_val)\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:32:39.684661Z","iopub.execute_input":"2022-09-10T14:32:39.685034Z","iopub.status.idle":"2022-09-10T14:32:44.449292Z","shell.execute_reply.started":"2022-09-10T14:32:39.684989Z","shell.execute_reply":"2022-09-10T14:32:44.447653Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric_report(y_val, pred_lr)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:50:14.041773Z","iopub.execute_input":"2022-09-10T14:50:14.042175Z","iopub.status.idle":"2022-09-10T14:50:14.072239Z","shell.execute_reply.started":"2022-09-10T14:50:14.042108Z","shell.execute_reply":"2022-09-10T14:50:14.071198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Hmmm...slightly worse.","metadata":{}},{"cell_type":"markdown","source":"# Ensemble","metadata":{}},{"cell_type":"code","source":"val_preds = (pred_lr + pred_gbm_bin + pred_val_rf)/3","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:40:47.749038Z","iopub.execute_input":"2022-09-10T14:40:47.749440Z","iopub.status.idle":"2022-09-10T14:40:47.755506Z","shell.execute_reply.started":"2022-09-10T14:40:47.749356Z","shell.execute_reply":"2022-09-10T14:40:47.754544Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_rmse = r_mse(val_preds , y_val)\nval_rmse","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:40:53.843555Z","iopub.execute_input":"2022-09-10T14:40:53.843915Z","iopub.status.idle":"2022-09-10T14:40:53.850749Z","shell.execute_reply.started":"2022-09-10T14:40:53.843864Z","shell.execute_reply":"2022-09-10T14:40:53.849805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(val_preds)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:40:59.341762Z","iopub.execute_input":"2022-09-10T14:40:59.342120Z","iopub.status.idle":"2022-09-10T14:40:59.348150Z","shell.execute_reply.started":"2022-09-10T14:40:59.342069Z","shell.execute_reply":"2022-09-10T14:40:59.347426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_preds_bin =[]\nfor val in val_preds:\n    if val >= 0.5:\n        val_preds_bin.append(1)\n    else:\n        val_preds_bin.append(0)\n   \n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:41:04.453087Z","iopub.execute_input":"2022-09-10T14:41:04.453530Z","iopub.status.idle":"2022-09-10T14:41:04.465902Z","shell.execute_reply.started":"2022-09-10T14:41:04.453437Z","shell.execute_reply":"2022-09-10T14:41:04.464626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"metric_report(y_val, val_preds_bin)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:50:50.024195Z","iopub.execute_input":"2022-09-10T14:50:50.024583Z","iopub.status.idle":"2022-09-10T14:50:50.084601Z","shell.execute_reply.started":"2022-09-10T14:50:50.024533Z","shell.execute_reply":"2022-09-10T14:50:50.083518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Classification report","metadata":{}},{"cell_type":"code","source":"target_names = ['signal', 'Background']\nprint(classification_report(y_val, val_preds_bin, target_names=target_names))\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:51:04.554017Z","iopub.execute_input":"2022-09-10T14:51:04.554449Z","iopub.status.idle":"2022-09-10T14:51:04.598729Z","shell.execute_reply.started":"2022-09-10T14:51:04.554384Z","shell.execute_reply":"2022-09-10T14:51:04.597691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"con_matrix(y_val, val_preds_bin)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:44:43.377445Z","iopub.execute_input":"2022-09-10T14:44:43.378076Z","iopub.status.idle":"2022-09-10T14:44:44.063486Z","shell.execute_reply.started":"2022-09-10T14:44:43.378010Z","shell.execute_reply":"2022-09-10T14:44:44.062200Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ROC Curve","metadata":{}},{"cell_type":"code","source":"roc_curvePlot(y_val, val_preds_bin)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T14:52:51.236817Z","iopub.execute_input":"2022-09-10T14:52:51.237228Z","iopub.status.idle":"2022-09-10T14:52:51.430783Z","shell.execute_reply.started":"2022-09-10T14:52:51.237164Z","shell.execute_reply":"2022-09-10T14:52:51.429720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Final predictions for the test set","metadata":{}},{"cell_type":"markdown","source":"### Here, just taking the Xgboost's results","metadata":{}},{"cell_type":"code","source":"X_test = test[selected_cols]\ncols = X_test.columns\nX_scaled_test = pd.DataFrame(sc.fit_transform(X_test), columns=cols)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:31:39.461144Z","iopub.execute_input":"2022-09-10T13:31:39.461595Z","iopub.status.idle":"2022-09-10T13:31:39.806885Z","shell.execute_reply.started":"2022-09-10T13:31:39.461531Z","shell.execute_reply":"2022-09-10T13:31:39.805789Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probs = xgb.predict(X_scaled_test)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:33:44.563239Z","iopub.execute_input":"2022-09-10T13:33:44.563806Z","iopub.status.idle":"2022-09-10T13:34:12.650504Z","shell.execute_reply.started":"2022-09-10T13:33:44.563757Z","shell.execute_reply":"2022-09-10T13:34:12.649651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_probs","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:34:12.651891Z","iopub.execute_input":"2022-09-10T13:34:12.652301Z","iopub.status.idle":"2022-09-10T13:34:12.657266Z","shell.execute_reply.started":"2022-09-10T13:34:12.652261Z","shell.execute_reply":"2022-09-10T13:34:12.656596Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(test_probs)","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:34:12.658426Z","iopub.execute_input":"2022-09-10T13:34:12.658899Z","iopub.status.idle":"2022-09-10T13:34:12.691842Z","shell.execute_reply.started":"2022-09-10T13:34:12.658856Z","shell.execute_reply":"2022-09-10T13:34:12.690914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submit to Kaggle","metadata":{}},{"cell_type":"code","source":"test.shape","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:34:12.699361Z","iopub.execute_input":"2022-09-10T13:34:12.699846Z","iopub.status.idle":"2022-09-10T13:34:12.711637Z","shell.execute_reply.started":"2022-09-10T13:34:12.699796Z","shell.execute_reply":"2022-09-10T13:34:12.710691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"p  = pd.read_csv(path+'test.csv.zip')","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:34:12.713320Z","iopub.execute_input":"2022-09-10T13:34:12.713844Z","iopub.status.idle":"2022-09-10T13:34:24.010994Z","shell.execute_reply.started":"2022-09-10T13:34:12.713799Z","shell.execute_reply":"2022-09-10T13:34:24.009862Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit = pd.DataFrame()\ndf_submit[\"id\"] = p[\"id\"] \ndf_submit[\"prediction\"] = test_probs","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:34:24.012493Z","iopub.execute_input":"2022-09-10T13:34:24.012815Z","iopub.status.idle":"2022-09-10T13:34:24.094109Z","shell.execute_reply.started":"2022-09-10T13:34:24.012767Z","shell.execute_reply":"2022-09-10T13:34:24.092779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_submit.to_csv('submission.csv', index=False, sep=',')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-10T13:34:24.095483Z","iopub.execute_input":"2022-09-10T13:34:24.095819Z","iopub.status.idle":"2022-09-10T13:34:25.333447Z","shell.execute_reply.started":"2022-09-10T13:34:24.095763Z","shell.execute_reply":"2022-09-10T13:34:25.332500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# End of Notebook","metadata":{}}]}