{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install mapie","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:42:49.728101Z","iopub.execute_input":"2022-11-10T14:42:49.728544Z","iopub.status.idle":"2022-11-10T14:43:00.565916Z","shell.execute_reply.started":"2022-11-10T14:42:49.728508Z","shell.execute_reply":"2022-11-10T14:43:00.564406Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport matplotlib.pyplot as plt\nimport seaborn as sbn\nfrom sklearn.ensemble import GradientBoostingRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import accuracy_score\nfrom sklearn import metrics\nfrom sklearn.preprocessing import StandardScaler\nimport shap\nimport math\nimport scipy.stats\nfrom mapie.regression import MapieRegressor\nfrom mapie.classification import MapieClassifier\nfrom mapie.metrics import regression_coverage_score\nfrom mapie.quantile_regression import MapieQuantileRegressor","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.569004Z","iopub.execute_input":"2022-11-10T14:43:00.569426Z","iopub.status.idle":"2022-11-10T14:43:00.577749Z","shell.execute_reply.started":"2022-11-10T14:43:00.569387Z","shell.execute_reply":"2022-11-10T14:43:00.576403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_data(path, with_header = True):\n    if(with_header):\n        data = pd.read_csv(path)\n    else:\n        data = pd.read_csv(path, header = None)\n    return data","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.579164Z","iopub.execute_input":"2022-11-10T14:43:00.579548Z","iopub.status.idle":"2022-11-10T14:43:00.591376Z","shell.execute_reply.started":"2022-11-10T14:43:00.579514Z","shell.execute_reply":"2022-11-10T14:43:00.590225Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Data information: https://www.kaggle.com/competitions/big-data-derby-2022/data","metadata":{}},{"cell_type":"markdown","source":"# DATA HORSES","metadata":{}},{"cell_type":"markdown","source":"Here is the raw data on horses :","metadata":{}},{"cell_type":"code","source":"data_horses = get_data('../input/big-data-derby-2022/nyra_start_table.csv', with_header = False)\ndata_horses.columns = ['track_id','race_date','race_number','program_number', 'weight_carried','jockey','odds',\n                  'position_at_finish']\ndata_horses","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.595212Z","iopub.execute_input":"2022-11-10T14:43:00.596215Z","iopub.status.idle":"2022-11-10T14:43:00.641192Z","shell.execute_reply.started":"2022-11-10T14:43:00.596175Z","shell.execute_reply":"2022-11-10T14:43:00.640271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With the raw data, a race is identified with the composition of *track_id*, *race_date* and *race_number*, we will\n:\n* create a new attribute *race_id* with those 3 information to identify a race uniquely\n* create a new attribute *id* combining *race_id* and *program_number*\n* divide the odds by 100 to derive the odds to 1\n\nThe creation of *race_id* and *id* will allow us to combine this data table with other data tables more easily.","metadata":{}},{"cell_type":"code","source":"def compute_race_id(data):\n    data['race_id'] = data.track_id + data.race_date + data.race_number.astype(str)\n    return data['race_id']\n\ndef compute_race_id_and_id(data):\n    data['race_id'] = compute_race_id(data)\n    data['id'] = data.race_id + data.program_number.astype(str)\n    return data\n\ndef compute_real_odds(odds):\n    return odds/100\n\ndef preprocess_horses_data(data_horses):\n    data_horses['odds'] = compute_real_odds(data_horses['odds'])\n    data_horses = compute_race_id_and_id(data_horses).sort_values(by='id').reset_index(drop=True)\n    return data_horses[['race_id', 'track_id', 'race_date', 'race_number', 'program_number', 'id', 'jockey', 'weight_carried', 'odds', 'position_at_finish']]\n\ndata_horses_preprocessed = preprocess_horses_data(data_horses)\ndata_horses_preprocessed","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.642636Z","iopub.execute_input":"2022-11-10T14:43:00.643207Z","iopub.status.idle":"2022-11-10T14:43:00.702172Z","shell.execute_reply.started":"2022-11-10T14:43:00.643170Z","shell.execute_reply":"2022-11-10T14:43:00.701105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With the original data there is no way to identify a horse from a race to another, but it is possible to get this info using the following dataset: https://www.kaggle.com/datasets/themarkgreen/big-data-derby-2022-global-horse-ids-and-places .","metadata":{}},{"cell_type":"code","source":"horse_ids = get_data('../input/big-data-derby-2022-global-horse-ids-and-places/horse_ids.csv')\nhorse_ids","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.703392Z","iopub.execute_input":"2022-11-10T14:43:00.703708Z","iopub.status.idle":"2022-11-10T14:43:00.736638Z","shell.execute_reply.started":"2022-11-10T14:43:00.703670Z","shell.execute_reply":"2022-11-10T14:43:00.735584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will use the attributes *track_id*, *race_date*, *race* and *program_number* of this table to compute an attribute *id* as we did for the table *data_horses*. Since each value of *id* of this table will correspond to the same value of *id* in the table *data_horses*, we will be able to add the attribute *horse_id* to *data_horses*, so that a horse can be identified from one race to another.\n\nHowever, we can notice that there is 14916 rows in the table *horse_ids* and 14915 in the table *data_horses*: we will sort both tables by *id* and delete the row of the table *horse_ids* that does not match any row in the table *data_horses*.","metadata":{}},{"cell_type":"code","source":"def add_horse_id(data_horses, horse_ids):\n    horse_ids['race_number'] = horse_ids['race']\n    horse_ids = compute_race_id_and_id(horse_ids).sort_values(by='id').reset_index(drop=True).drop(index=7557).reset_index(drop=True)\n    data_horses['horse_id'] = horse_ids['horse_id']\n    return data_horses[['race_id', 'track_id', 'race_date', 'race_number', 'program_number', 'horse_id', 'jockey', 'weight_carried', 'odds', 'position_at_finish']]\n\ndata_horses = add_horse_id(data_horses_preprocessed, horse_ids)\ndata_horses","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.737893Z","iopub.execute_input":"2022-11-10T14:43:00.738188Z","iopub.status.idle":"2022-11-10T14:43:00.802968Z","shell.execute_reply.started":"2022-11-10T14:43:00.738160Z","shell.execute_reply":"2022-11-10T14:43:00.801865Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now we explore a bit the data:\n* How many races are there ?\n* How many jockeys ?\n* How many horses ?\n* When is the first race ? And the last one ?\n* How does the boxplot of *weight_carried* and *odds* look like ?\n\nSpecifically, we highlight the correlation between the odds and the final position, showing the boxplot and the distribution of odds for each final position.","metadata":{}},{"cell_type":"code","source":"def explore_data_horses(data_horses):\n    print('Nb races : ' + str(len(np.unique(data_horses['race_id']))))\n    print('Nb jockeys : ' + str(len(np.unique(data_horses['jockey']))))\n    print('Nb horses : ' + str(len(np.unique(data_horses['horse_id']))))\n    print(('First race : ' + str(data_horses['race_date'].min())))\n    print(('Last race : ' + str(data_horses['race_date'].max())))\n\n    plt.boxplot(data_horses['weight_carried'])\n    plt.title('weigh carried')\n    plt.show()\n\n    plt.boxplot(data_horses['odds'])\n    plt.title('odds')\n    plt.show()\n\n    sbn.catplot(x = 'position_at_finish', y = 'odds', kind = \"box\", data = data_horses)\n    plt.ylim(0,90)\n    plt.title('odds by position at finish')\n    plt.show()\n\n    data_horses.groupby('position_at_finish')['odds'].apply(lambda x: sbn.distplot(x, hist=False))\n    plt.xlabel('odds')\n    plt.ylabel(\"\")\n    plt.xlim(-2,15)\n    plt.legend(labels=['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14'])\n    plt.title('Odds distribution for each final position')\n    plt.show()\n    return True\n\nexplore_data_horses(data_horses)","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:00.804478Z","iopub.execute_input":"2022-11-10T14:43:00.805098Z","iopub.status.idle":"2022-11-10T14:43:02.104367Z","shell.execute_reply.started":"2022-11-10T14:43:00.805064Z","shell.execute_reply":"2022-11-10T14:43:02.103358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATA RACES","metadata":{}},{"cell_type":"markdown","source":"Here is the raw data on races :","metadata":{}},{"cell_type":"code","source":"data_races = get_data('../input/big-data-derby-2022/nyra_race_table.csv')\ndata_races","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:02.105679Z","iopub.execute_input":"2022-11-10T14:43:02.106270Z","iopub.status.idle":"2022-11-10T14:43:02.148052Z","shell.execute_reply.started":"2022-11-10T14:43:02.106233Z","shell.execute_reply":"2022-11-10T14:43:02.146874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As before, we will compute an attribute *race_id* by combining the attributes *track_id*, *race_date* and *race_number*, it will allow us to combine the information of the data tables *data_races* and *data_horses*. We will also convert the attributes *distance_id* and *run_up_distance* to meters and sum them to get the total distance of each race. We only keep the attributes that are not already available in the data table *data_horses*.","metadata":{}},{"cell_type":"code","source":"def convert_distance_id_to_meters(distance_id):\n    return (distance_id/100)*201.168\n\ndef convert_run_up_distance_to_meters(run_up_distance):\n    return run_up_distance*0.3048\n\ndef compute_full_distance_in_meters(data):\n    data['distance_id'] = convert_distance_id_to_meters(data.distance_id)\n    data['run_up_distance'] = convert_run_up_distance_to_meters(data.run_up_distance)\n    data['total_distance'] = data['distance_id'] + data['run_up_distance']\n    # drop distance_id and run_up_distance since it is now useless\n    return data.drop(columns=['distance_id', 'run_up_distance'])\n\ndef preprocess_races_data(data_races):\n    data_races['race_id'] = compute_race_id(data_races)\n    data_races = compute_full_distance_in_meters(data_races)\n    return data_races[['race_id', 'course_type', 'track_condition', 'race_type', 'purse', 'total_distance']]\n\ndata_races = preprocess_races_data(data_races)\ndata_races","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:02.151356Z","iopub.execute_input":"2022-11-10T14:43:02.151672Z","iopub.status.idle":"2022-11-10T14:43:02.180433Z","shell.execute_reply.started":"2022-11-10T14:43:02.151643Z","shell.execute_reply":"2022-11-10T14:43:02.179393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We explore a bit the data (see https://www.kaggle.com/competitions/big-data-derby-2022/data for more information about the values of each attribute).","metadata":{}},{"cell_type":"code","source":"def explore_data_races(data_races):\n    plt.boxplot(data_races['total_distance'])\n    plt.title('Distance per race in meters')\n    plt.show()\n\n    plt.boxplot(data_races['purse'])\n    plt.title('Purse in dollars')\n    plt.ylim(0,2e5)\n    plt.show()\n\n    data_races['course_type'].value_counts().plot(kind='pie')\n    plt.title('Course type')\n    plt.show()\n\n    data_races['track_condition'].value_counts().plot(kind='pie')\n    plt.title('Conditions of the race')\n    plt.show()\n\n    data_races['race_type'].value_counts().plot(kind='pie')\n    plt.title('Race type')\n    plt.show()\n    return True\n\nexplore_data_races(data_races)","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:02.181694Z","iopub.execute_input":"2022-11-10T14:43:02.182022Z","iopub.status.idle":"2022-11-10T14:43:03.138503Z","shell.execute_reply.started":"2022-11-10T14:43:02.181992Z","shell.execute_reply":"2022-11-10T14:43:03.137244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# COMBINING DATA HORSES AND DATA RACES","metadata":{}},{"cell_type":"markdown","source":"* For each value of *race_id* in *data_horses* we add the values of the corresponding row in *data_races* (first sorting both tables by *race_id* to reduce the complexity).\n* We also compute the mean, min, max and standard deviation of the odds of each race and add it to the information of the race.\n* We compute the number of horses of each race, since it is not always the same number.\n* We then delete the attribute *race_id* since it was an artificial attribute computed only in order to combine both tables easily.","metadata":{}},{"cell_type":"code","source":"def compute_mean_odds(data_horses):\n    mean_odds_list = []\n    for mean_odds in data_horses.groupby('race_id')['odds'].mean():\n        mean_odds_list.append(mean_odds)\n    return mean_odds_list\n\ndef compute_std_odds(data_horses):\n    std_odds_list = []\n    for std_odds in data_horses.groupby('race_id')['odds'].agg(np.std):\n        std_odds_list.append(std_odds)\n    return std_odds_list\n\ndef compute_min_odds(data_horses):\n    min_odds_list = []\n    for min_odds in data_horses.groupby('race_id')['odds'].min():\n        min_odds_list.append(min_odds)\n    return min_odds_list\n\ndef compute_max_odds(data_horses):\n    max_odds_list = []\n    for max_odds in data_horses.groupby('race_id')['odds'].max():\n        max_odds_list.append(max_odds)\n    return max_odds_list\n\ndef add_nb_horses(data_horses):\n    nb_horses_in_race = data_horses['race_id'].value_counts()\n    nb_horses = []\n    for index in data_horses.index:\n        nb_horses.append(nb_horses_in_race[data_horses['race_id'][index]])\n    data_horses['nb_horses'] = nb_horses\n    return data_horses\n\n\ndef add_race_info_to_horses_data(data_horses, data_races):\n    data_races['mean_odds'] = compute_mean_odds(data_horses)\n    data_races['std_odds'] = compute_std_odds(data_horses)\n    data_races['min_odds'] = compute_min_odds(data_horses)\n    data_races['max_odds'] = compute_max_odds(data_horses)\n    \n    course_type = []\n    track_condition = []\n    race_type = []\n    purse = []\n    total_distance = []\n    mean_odds = []\n    std_odds = []\n    min_odds = []\n    max_odds = []\n    \n    index_race = 0\n    for index_horse in data_horses.index:\n        found = False\n        while found == False:\n            if data_horses['race_id'][index_horse] == data_races['race_id'][index_race]:\n                course_type.append(data_races['course_type'][index_race])\n                track_condition.append(data_races['track_condition'][index_race])\n                race_type.append(data_races['race_type'][index_race])\n                purse.append(data_races['purse'][index_race])\n                total_distance.append(data_races['total_distance'][index_race])\n                mean_odds.append(data_races['mean_odds'][index_race])\n                std_odds.append(data_races['std_odds'][index_race])\n                min_odds.append(data_races['min_odds'][index_race])\n                max_odds.append(data_races['max_odds'][index_race])\n                found = True\n            else:\n                index_race = index_race + 1\n    data_horses['course_type'] = course_type\n    data_horses['track_condition'] = track_condition\n    data_horses['race_type'] = race_type\n    data_horses['purse'] = purse\n    data_horses['total_distance'] = total_distance\n    data_horses['mean_odds'] = mean_odds\n    data_horses['std_odds'] = std_odds\n    data_horses['min_odds'] = min_odds\n    data_horses['max_odds'] = max_odds\n    \n    data_horses = add_nb_horses(data_horses)\n    return data_horses[['track_id', 'race_date', 'race_number', 'course_type', 'track_condition',\n                        'race_type', 'purse', 'total_distance', 'nb_horses', 'mean_odds', 'min_odds', 'max_odds', 'std_odds',\n                        'program_number', 'horse_id', 'jockey', 'weight_carried', 'odds', 'position_at_finish']]\n\ndata_full = add_race_info_to_horses_data(data_horses.sort_values(by='race_id').reset_index(drop=True),\n                                         data_races.sort_values(by='race_id').reset_index(drop=True)\n                                        )\ndata_full","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:03.140737Z","iopub.execute_input":"2022-11-10T14:43:03.141537Z","iopub.status.idle":"2022-11-10T14:43:04.559877Z","shell.execute_reply.started":"2022-11-10T14:43:03.141489Z","shell.execute_reply":"2022-11-10T14:43:04.558961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# FINAL POSITION PREDICTION","metadata":{}},{"cell_type":"markdown","source":"We label encode each categorical attribute.","metadata":{}},{"cell_type":"code","source":"def label_encoding(data, label):\n    data[label] = data[label].astype('category')\n    data[label] = data[label].cat.codes\n    return data\n\ndef preprocessing(data):\n    labels_to_encode = ['track_id', 'race_type', 'track_condition', 'course_type', 'program_number', 'jockey']\n    for label in labels_to_encode:\n        data = label_encoding(data, label)\n    return data\n\ndata_full = preprocessing(data_full)\ndata_full","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:04.561363Z","iopub.execute_input":"2022-11-10T14:43:04.561958Z","iopub.status.idle":"2022-11-10T14:43:04.609483Z","shell.execute_reply.started":"2022-11-10T14:43:04.561924Z","shell.execute_reply":"2022-11-10T14:43:04.608277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following matrix *X* will be the input of the ML model.","metadata":{}},{"cell_type":"code","source":"X = data_full.drop(columns=['position_at_finish', 'race_date'])\nX","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:04.610721Z","iopub.execute_input":"2022-11-10T14:43:04.611045Z","iopub.status.idle":"2022-11-10T14:43:04.643544Z","shell.execute_reply.started":"2022-11-10T14:43:04.611016Z","shell.execute_reply":"2022-11-10T14:43:04.642472Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The following vector *y* will be the output the ML model will try to predict.","metadata":{}},{"cell_type":"code","source":"y = data_full['position_at_finish']\ny","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:04.644785Z","iopub.execute_input":"2022-11-10T14:43:04.645124Z","iopub.status.idle":"2022-11-10T14:43:04.654100Z","shell.execute_reply.started":"2022-11-10T14:43:04.645094Z","shell.execute_reply":"2022-11-10T14:43:04.652927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The task of predicting *y* can be seen either as a classification task with 14 possible labels from 1 to 14, or as a regression task (selecting as the position predicted the closest integer to the value returned). The regression option will be chosen since the possible positions have an order (the model has to manage the fact that if the true position is 3, then predicting the position 2 is better than predicting the position 9).","metadata":{}},{"cell_type":"markdown","source":"Let's train the model with the gradient boosting regressor of the sklearn library. We also plot the shapley values (https://shap.readthedocs.io/en/latest/example_notebooks/overviews/An%20introduction%20to%20explainable%20AI%20with%20Shapley%20values.html) in order to visualize the impact of each attribute on the model output.","metadata":{}},{"cell_type":"code","source":"def train_and_show_results_with_shap_values(X, y, regressor, test_size):\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size)\n\n    regressor.fit(X_train, y_train)\n    y_test_pred = regressor.predict(X_test)\n\n    explainer = shap.Explainer(regressor, X_train)\n    shap_values = explainer(X_train)\n    shap.summary_plot(shap_values, X_train, plot_type=\"layered_violin\", color='coolwarm')\n\n    print('Regressor accuracy (with prediction rounded to nearest integer): '+ str(accuracy_score(y_test, np.round(y_test_pred))))\n    return True\n\ntrain_and_show_results_with_shap_values(X, y, GradientBoostingRegressor(),  0.2)","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:04.655425Z","iopub.execute_input":"2022-11-10T14:43:04.655765Z","iopub.status.idle":"2022-11-10T14:43:26.544466Z","shell.execute_reply.started":"2022-11-10T14:43:04.655735Z","shell.execute_reply":"2022-11-10T14:43:26.543272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It can be noticed that the accuracy of the model for predicting the exact position of a horse before the race is not really good (~17%) and that, obviously, the odds have a high impact on the model output.","metadata":{}},{"cell_type":"markdown","source":"# CONFORMAL PREDICTION OF THE POSITION","metadata":{}},{"cell_type":"markdown","source":"The mapie library (https://mapie.readthedocs.io/en/latest/) will be used in order to compute the conformal prediction of the model (i.e a confidence interval of the model, with a confidence level to be defined). For example, the mapie regressor will return the interval [2,6] with a confidence level of 90%, meaning that the true value will be in this interval with a confidence level of 90%. The confidence level will be defined as 80% (alpha = 0.2) but can be changed.","metadata":{}},{"cell_type":"code","source":"def conformal_learning(X, y, alpha, calibration_size, test_size):\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size)\n    X_train, X_calib, y_train, y_calib = train_test_split(X_train, y_train, test_size = calibration_size)\n    mapie_reg = MapieQuantileRegressor(alpha = alpha).fit(\n        X_train,\n        y_train,\n        X_calib = X_calib,\n        y_calib = y_calib\n    )\n    y_test_pred, y_pred_interval = mapie_reg.predict(X_test)\n    y_lower_bound = y_pred_interval[:,0]\n    y_upper_bound = y_pred_interval[:,1]\n\n    print('Mean size of the interval predicted for a confidence level of ' + str((1-alpha)*100) + '% : ' + str((y_upper_bound - y_lower_bound).mean()))\n    print('Median size of the interval predicted for a confidence level of ' + str((1-alpha)*100) + '% : ' + str(np.median(y_upper_bound - y_lower_bound)))\n    print('Min size of the interval predicted for a confidence level of ' + str((1-alpha)*100) + '% : ' + str((y_upper_bound - y_lower_bound).min()))\n    print('Max size of the interval predicted for a confidence level of ' + str((1-alpha)*100) + '% : ' + str((y_upper_bound - y_lower_bound).max()))\n\n    plt.hist(y_upper_bound - y_lower_bound)\n    plt.title('Histogram of the size of the interval predicted for a confidence level ofso ' + str((1-alpha)*100) + '%')\n    plt.show() \n    return True\n\nconformal_learning(X, y, alpha = 0.2, calibration_size = 0.1, test_size = 0.2)","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:43:26.546104Z","iopub.execute_input":"2022-11-10T14:43:26.546893Z","iopub.status.idle":"2022-11-10T14:44:40.790140Z","shell.execute_reply.started":"2022-11-10T14:43:26.546847Z","shell.execute_reply":"2022-11-10T14:44:40.789090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It can be noticed that in mean the size of the interval predicted for a confidence level of 80% is a bit more than 5, wich means for example a prediction such as [2:7]. The histogram shows that there are few cases in wich the model can give an accurate prediction (size of the interval predicted smaller than 3). There is a trade-off between the accuracy of the predictions (the size of the interval predicted) and the confidence level: in order to get smaller intervals the confidence level has to be lowered.","metadata":{}},{"cell_type":"markdown","source":"# PREDICTION WITH A MARGIN","metadata":{}},{"cell_type":"markdown","source":"Instead of defining a confidence level that allow to compute a confidence interval, we can choose to define a margin that seems acceptable and compute the accuracy of the model taking this margin into account. For instance, a +1 -1 margin can be defined, so that if the model predict the position 5 and the actual value is 4 or 6 we consider that the prediction is successful.","metadata":{}},{"cell_type":"code","source":"# fonction that will allow to compute the accuracy of the prediction with a margin to define (default margin = 1)\ndef compute_accuracy_range_prediction(y, y_preds, margin=1):\n    preds_with_margin = []\n    true_is_in_predicted_range = []\n    for index, true in enumerate(y):\n        pred = int(np.round(y_preds[index]))\n        preds_with_margin.append(range(pred - margin, pred + margin + 1))\n        true_is_in_predicted_range.append(true in preds_with_margin[index])\n    return accuracy_score(np.full(np.shape(true_is_in_predicted_range), True), true_is_in_predicted_range)","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:44:40.792016Z","iopub.execute_input":"2022-11-10T14:44:40.792599Z","iopub.status.idle":"2022-11-10T14:44:40.799055Z","shell.execute_reply.started":"2022-11-10T14:44:40.792562Z","shell.execute_reply":"2022-11-10T14:44:40.797821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_and_show_accuracy_with_margin(X, y, regressor, test_size, margin):\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = test_size)\n\n    regressor.fit(X_train, y_train)\n    y_test_pred = regressor.predict(X_test)\n\n    print('Regressor accuracy (with margin error of ' + str(margin) +'): '+ str(compute_accuracy_range_prediction(y_test, y_test_pred, margin=margin)))\n    return True\n\ntrain_and_show_accuracy_with_margin(X, y, GradientBoostingRegressor(),  test_size = 0.2, margin = 1)","metadata":{"execution":{"iopub.status.busy":"2022-11-10T14:44:40.800338Z","iopub.execute_input":"2022-11-10T14:44:40.801226Z","iopub.status.idle":"2022-11-10T14:44:43.436660Z","shell.execute_reply.started":"2022-11-10T14:44:40.801191Z","shell.execute_reply":"2022-11-10T14:44:43.435474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"With a +1 -1 error margin, the accuracy of the predictions increases until a bit more than 50%.","metadata":{}}]}