{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T05:18:12.831730Z","iopub.execute_input":"2022-08-29T05:18:12.832487Z","iopub.status.idle":"2022-08-29T05:18:12.866496Z","shell.execute_reply.started":"2022-08-29T05:18:12.832385Z","shell.execute_reply":"2022-08-29T05:18:12.865377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport seaborn as sns\nimport numpy as np\nimport matplotlib.pyplot as plt","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T05:18:12.869377Z","iopub.execute_input":"2022-08-29T05:18:12.869841Z","iopub.status.idle":"2022-08-29T05:18:14.228194Z","shell.execute_reply.started":"2022-08-29T05:18:12.869795Z","shell.execute_reply":"2022-08-29T05:18:14.227347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# <span style=\"color:#FFC300;\">***Derby 2022***<span>\n_**<span style=\"color:#009dc7;\">I will be grateful if you rate this analysis</span>**_  🖤\n## <span style=\"color:#FFC300;\">**Plan**<span>\n<a id=\"table-of-contents\"></a>\n- [1. Basic EDA](#1)\n- [2. Race analysis](#2)\n- [3. Parameters of the race track](#3)\n- [4. Analysis hypotheses of categorical features](#4)  \n- [5. Analysis hypotheses of numetrical features](#5)     \n- [6. Map](#6)\n- [7. The importance of features](#7)","metadata":{}},{"cell_type":"markdown","source":"# **Basic EDA** <a id=\"1\"></a>","metadata":{}},{"cell_type":"markdown","source":"**The agenda**","metadata":{}},{"cell_type":"markdown","source":"The goal of this competition is to analyze horse racing tactics, drafting strategies, and path efficiency. You will develop a model using never-before-released coordinate data along with basic race information.","metadata":{}},{"cell_type":"code","source":"nyra_2019_data = pd.read_csv('../input/big-data-derby-2022/nyra_2019_complete.csv')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T05:18:14.229393Z","iopub.execute_input":"2022-08-29T05:18:14.229865Z","iopub.status.idle":"2022-08-29T05:18:31.454493Z","shell.execute_reply.started":"2022-08-29T05:18:14.229834Z","shell.execute_reply":"2022-08-29T05:18:31.453174Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"functions","metadata":{}},{"cell_type":"code","source":"def nan_zero_info(data):\n    print(f'нулей в датасете найдено\\n:{data.isnull().sum()}\\n')\n    print(f'NaN в датасете:\\n {data.isna().sum()}\\n')\n    print(f\"Типы  в датасете:\\n {data.dtypes}\\n\")","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:55:32.618420Z","iopub.execute_input":"2022-08-13T11:55:32.618857Z","iopub.status.idle":"2022-08-13T11:55:32.624834Z","shell.execute_reply.started":"2022-08-13T11:55:32.618825Z","shell.execute_reply":"2022-08-13T11:55:32.623752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def two_histograms(x, y):\n    x.hist(alpha=0.5, weights=[1./len(x)]*len(x),bins = 7)\n    y.hist(alpha=0.5, weights=[1./len(y)]*len(y),bins = 7)\n    plt.axvline(x.mean(), color='red', alpha=0.8, linestyle='dashed')\n    plt.axvline(y.mean(), color='blue', alpha=0.8, linestyle='dashed')","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:59:55.870275Z","iopub.execute_input":"2022-08-13T11:59:55.871390Z","iopub.status.idle":"2022-08-13T11:59:55.878950Z","shell.execute_reply.started":"2022-08-13T11:59:55.871311Z","shell.execute_reply":"2022-08-13T11:59:55.877554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**data**","metadata":{}},{"cell_type":"code","source":"nyra_2019_data","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:55:32.626467Z","iopub.execute_input":"2022-08-13T11:55:32.626819Z","iopub.status.idle":"2022-08-13T11:55:32.678506Z","shell.execute_reply.started":"2022-08-13T11:55:32.626771Z","shell.execute_reply":"2022-08-13T11:55:32.677251Z"},"_kg_hide-input":false,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_data.tail(10)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:55:32.681037Z","iopub.execute_input":"2022-08-13T11:55:32.681405Z","iopub.status.idle":"2022-08-13T11:55:32.703548Z","shell.execute_reply.started":"2022-08-13T11:55:32.681373Z","shell.execute_reply":"2022-08-13T11:55:32.702204Z"},"_kg_hide-input":false,"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**info**","metadata":{}},{"cell_type":"code","source":"nan_zero_info(nyra_2019_data)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:55:32.704975Z","iopub.execute_input":"2022-08-13T11:55:32.705408Z","iopub.status.idle":"2022-08-13T11:55:35.914489Z","shell.execute_reply.started":"2022-08-13T11:55:32.705369Z","shell.execute_reply":"2022-08-13T11:55:35.913196Z"},"_kg_hide-input":false,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1 There are no passes.\n\n2 not all data types are correct. Let's fix it","metadata":{}},{"cell_type":"markdown","source":"**Remove unnecessary spaces**","metadata":{}},{"cell_type":"code","source":"nyra_2019_data['program_number'] = nyra_2019_data['program_number'].replace(r'\\s+','',regex=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:55:35.916000Z","iopub.execute_input":"2022-08-13T11:55:35.916454Z","iopub.status.idle":"2022-08-13T11:55:44.329284Z","shell.execute_reply.started":"2022-08-13T11:55:35.916420Z","shell.execute_reply":"2022-08-13T11:55:44.328017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**replace strings with values**","metadata":{}},{"cell_type":"code","source":"nyra_2019_data['program_number'] = nyra_2019_data['program_number'].replace(r'1A','17',regex=True)\nnyra_2019_data['program_number'] = nyra_2019_data['program_number'].replace(r'2B','18',regex=True)\nnyra_2019_data['program_number'] = nyra_2019_data['program_number'].replace(r'3X','19',regex=True)\nnyra_2019_data['program_number'] = nyra_2019_data['program_number'].replace(r'1X','20',regex=True)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:55:44.330479Z","iopub.execute_input":"2022-08-13T11:55:44.330804Z","iopub.status.idle":"2022-08-13T11:56:13.829505Z","shell.execute_reply.started":"2022-08-13T11:55:44.330775Z","shell.execute_reply":"2022-08-13T11:56:13.828122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_data['program_number'] = nyra_2019_data['program_number'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:56:13.830936Z","iopub.execute_input":"2022-08-13T11:56:13.831273Z","iopub.status.idle":"2022-08-13T11:56:14.349668Z","shell.execute_reply.started":"2022-08-13T11:56:13.831242Z","shell.execute_reply":"2022-08-13T11:56:14.348395Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"check","metadata":{}},{"cell_type":"code","source":"nyra_2019_data.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:56:14.353185Z","iopub.execute_input":"2022-08-13T11:56:14.354191Z","iopub.status.idle":"2022-08-13T11:56:14.376520Z","shell.execute_reply.started":"2022-08-13T11:56:14.354149Z","shell.execute_reply":"2022-08-13T11:56:14.375342Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**basic statistics**","metadata":{}},{"cell_type":"code","source":"nyra_2019_data.describe().round(decimals = 2)","metadata":{"execution":{"iopub.status.busy":"2022-08-13T11:56:14.378899Z","iopub.execute_input":"2022-08-13T11:56:14.379493Z","iopub.status.idle":"2022-08-13T11:56:16.164424Z","shell.execute_reply.started":"2022-08-13T11:56:14.379445Z","shell.execute_reply":"2022-08-13T11:56:16.162992Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1 The average price for horse racing is 62000.The maximum price is 1500000\n\n2 Usually horse racing takes place during the day\n\n3 The average weight carried at a horse race is equal to 120. Max is 160","metadata":{}},{"cell_type":"markdown","source":"**correlation**","metadata":{}},{"cell_type":"code","source":"sns.heatmap(nyra_2019_data.corr())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:07:57.796522Z","iopub.execute_input":"2022-08-13T12:07:57.796938Z","iopub.status.idle":"2022-08-13T12:08:00.284940Z","shell.execute_reply.started":"2022-08-13T12:07:57.796906Z","shell.execute_reply":"2022-08-13T12:08:00.283694Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1 The distance_id and weight_carrier features are correlated on average.\n\n2 Other features have a weak connection","metadata":{}},{"cell_type":"markdown","source":"**Basic information**","metadata":{}},{"cell_type":"code","source":"for name_col in nyra_2019_data.columns:\n    print(f'The most frequent value in the data ({name_col}): {nyra_2019_data[name_col].value_counts().idxmax()}')","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:08:38.769937Z","iopub.execute_input":"2022-08-13T12:08:38.770877Z","iopub.status.idle":"2022-08-13T12:08:44.156609Z","shell.execute_reply.started":"2022-08-13T12:08:38.770832Z","shell.execute_reply":"2022-08-13T12:08:44.155305Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's divide the data by type","metadata":{}},{"cell_type":"code","source":"cat_cols, num_cols = nyra_2019_data.dtypes[nyra_2019_data.dtypes == 'object'].keys(), nyra_2019_data.dtypes[nyra_2019_data.dtypes != 'object'].keys()\n\nnum_data = nyra_2019_data.select_dtypes(np.number)\ncat_data = nyra_2019_data.select_dtypes('object')","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:08:51.581277Z","iopub.execute_input":"2022-08-13T12:08:51.581728Z","iopub.status.idle":"2022-08-13T12:08:52.129369Z","shell.execute_reply.started":"2022-08-13T12:08:51.581694Z","shell.execute_reply":"2022-08-13T12:08:52.127796Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pandas_profiling import ProfileReport\ndata = ProfileReport(nyra_2019_data, minimal=True)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T05:18:45.163924Z","iopub.execute_input":"2022-08-29T05:18:45.164369Z","iopub.status.idle":"2022-08-29T05:18:46.629040Z","shell.execute_reply.started":"2022-08-29T05:18:45.164336Z","shell.execute_reply":"2022-08-29T05:18:46.627729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data.to_widgets()","metadata":{"execution":{"iopub.status.busy":"2022-08-29T05:18:46.631029Z","iopub.execute_input":"2022-08-29T05:18:46.631565Z","iopub.status.idle":"2022-08-29T05:19:43.909495Z","shell.execute_reply.started":"2022-08-29T05:18:46.631531Z","shell.execute_reply":"2022-08-29T05:19:43.907828Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install anai-opensource==0.1.2-alpha-5\n\nimport anai\nfrom anai.preprocessing import Preprocessor\nimport plotly.express as px","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-29T05:21:31.107697Z","iopub.execute_input":"2022-08-29T05:21:31.108145Z","iopub.status.idle":"2022-08-29T05:22:25.436429Z","shell.execute_reply.started":"2022-08-29T05:21:31.108112Z","shell.execute_reply":"2022-08-29T05:22:25.435038Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Race analysis**  <a id=\"2\"></a>","metadata":{}},{"cell_type":"markdown","source":"Let's analyze some of the features related to race","metadata":{}},{"cell_type":"code","source":"nyra_2019_data['race_number'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:08:57.853015Z","iopub.execute_input":"2022-08-13T12:08:57.853490Z","iopub.status.idle":"2022-08-13T12:08:57.906741Z","shell.execute_reply.started":"2022-08-13T12:08:57.853454Z","shell.execute_reply":"2022-08-13T12:08:57.905481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = nyra_2019_data, x = 'race_number')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:09:05.616483Z","iopub.execute_input":"2022-08-13T12:09:05.616915Z","iopub.status.idle":"2022-08-13T12:09:06.217583Z","shell.execute_reply.started":"2022-08-13T12:09:05.616881Z","shell.execute_reply":"2022-08-13T12:09:06.216382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1  Most often the competitions take place on the route 8 6 and 7\n\n2 Less often than not, horse races take place on route 13","metadata":{}},{"cell_type":"markdown","source":"Consider the most frequent strip for racing","metadata":{}},{"cell_type":"markdown","source":"# The race number 8","metadata":{}},{"cell_type":"code","source":"race_8 = nyra_2019_data.loc[nyra_2019_data['race_number'] == 8]\nrace_8","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:09:11.364769Z","iopub.execute_input":"2022-08-13T12:09:11.365169Z","iopub.status.idle":"2022-08-13T12:09:11.477421Z","shell.execute_reply.started":"2022-08-13T12:09:11.365137Z","shell.execute_reply":"2022-08-13T12:09:11.476178Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_8['jockey'].value_counts().loc[ lambda x: x > 35000]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:09:43.277347Z","iopub.execute_input":"2022-08-13T12:09:43.278462Z","iopub.status.idle":"2022-08-13T12:09:43.351665Z","shell.execute_reply.started":"2022-08-13T12:09:43.278417Z","shell.execute_reply":"2022-08-13T12:09:43.350778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most often he drives along the highway  Manuel Franco","metadata":{}},{"cell_type":"code","source":"course_type = list(race_8[\"course_type\"].value_counts())\nlabel_course_type = list(pd.unique(race_8[\"course_type\"]))\nfig,ax1 = plt.subplots()\nax1.pie(course_type, labels=label_course_type, radius=1.2)\nax1.set_title('Course type distribution')\n\nrace_type = list(race_8[\"race_type\"].value_counts())\nlabel_race_type = list(pd.unique(race_8[\"race_type\"]))\nfig,ax2 = plt.subplots()\nax2.set_title('Race_type distribution')\nax2.pie(race_type, labels=label_race_type, radius=1.2)\n\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:12:58.362643Z","iopub.execute_input":"2022-08-13T12:12:58.363098Z","iopub.status.idle":"2022-08-13T12:12:58.996258Z","shell.execute_reply.started":"2022-08-13T12:12:58.363061Z","shell.execute_reply":"2022-08-13T12:12:58.990475Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Route 8 is diverse\n\n1 Basically, the races are on the lawn\n\n2  Main classification of the race is CLM - Claiming","metadata":{}},{"cell_type":"code","source":"race_8['program_number'].value_counts().loc[ lambda x: x > 35000]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:13:10.877603Z","iopub.execute_input":"2022-08-13T12:13:10.877998Z","iopub.status.idle":"2022-08-13T12:13:10.893634Z","shell.execute_reply.started":"2022-08-13T12:13:10.877967Z","shell.execute_reply":"2022-08-13T12:13:10.892567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most often , the horse number 4 ran along the track","metadata":{}},{"cell_type":"code","source":"race_8['post_time'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:13:13.436184Z","iopub.execute_input":"2022-08-13T12:13:13.437002Z","iopub.status.idle":"2022-08-13T12:13:13.456710Z","shell.execute_reply.started":"2022-08-13T12:13:13.436960Z","shell.execute_reply":"2022-08-13T12:13:13.455405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.histplot(data = race_8, x = 'post_time')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:13:27.420607Z","iopub.execute_input":"2022-08-13T12:13:27.421148Z","iopub.status.idle":"2022-08-13T12:13:27.943846Z","shell.execute_reply.started":"2022-08-13T12:13:27.421100Z","shell.execute_reply":"2022-08-13T12:13:27.942905Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The races take place many times a day.Morning afternoon and evening","metadata":{}},{"cell_type":"markdown","source":"# The race number 13","metadata":{}},{"cell_type":"code","source":"race_13 = nyra_2019_data.loc[nyra_2019_data['race_number'] == 13]\nrace_13","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:13:43.365965Z","iopub.execute_input":"2022-08-13T12:13:43.366438Z","iopub.status.idle":"2022-08-13T12:13:43.401405Z","shell.execute_reply.started":"2022-08-13T12:13:43.366401Z","shell.execute_reply":"2022-08-13T12:13:43.400137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_13['jockey'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:14:06.356454Z","iopub.execute_input":"2022-08-13T12:14:06.356856Z","iopub.status.idle":"2022-08-13T12:14:06.368290Z","shell.execute_reply.started":"2022-08-13T12:14:06.356824Z","shell.execute_reply":"2022-08-13T12:14:06.366181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"most often Jose L. Ortiz ran on the highway","metadata":{}},{"cell_type":"code","source":"course_type = list(race_13[\"course_type\"].value_counts())\nlabel_course_type = list(pd.unique(race_13[\"course_type\"]))\nfig,ax1 = plt.subplots()\nax1.pie(course_type, labels=label_course_type, radius=1.1)\nax1.set_title('Course type distribution')\n\nrace_type = list(race_13[\"race_type\"].value_counts())\nlabel_race_type = list(pd.unique(race_13[\"race_type\"]))\nfig,ax2 = plt.subplots()\nax2.pie(race_type, labels=label_race_type, radius=1.1)\nax2.set_title('Race_type distribution')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:14:50.485329Z","iopub.execute_input":"2022-08-13T12:14:50.485767Z","iopub.status.idle":"2022-08-13T12:14:50.698540Z","shell.execute_reply.started":"2022-08-13T12:14:50.485734Z","shell.execute_reply":"2022-08-13T12:14:50.697066Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"There are only 2 types of classifications and types of horse racing","metadata":{}},{"cell_type":"code","source":"race_13['program_number'].value_counts().loc[ lambda x: x > 500]","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:00.048374Z","iopub.execute_input":"2022-08-13T12:15:00.048787Z","iopub.status.idle":"2022-08-13T12:15:00.059281Z","shell.execute_reply.started":"2022-08-13T12:15:00.048746Z","shell.execute_reply":"2022-08-13T12:15:00.058335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Horse number 3 ran along the track more often","metadata":{}},{"cell_type":"code","source":"sns.histplot(data = race_13, x = 'post_time')","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:06.364282Z","iopub.execute_input":"2022-08-13T12:15:06.364687Z","iopub.status.idle":"2022-08-13T12:15:06.618548Z","shell.execute_reply.started":"2022-08-13T12:15:06.364655Z","shell.execute_reply":"2022-08-13T12:15:06.617376Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_13['post_time'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:14.689142Z","iopub.execute_input":"2022-08-13T12:15:14.689604Z","iopub.status.idle":"2022-08-13T12:15:14.699707Z","shell.execute_reply.started":"2022-08-13T12:15:14.689561Z","shell.execute_reply":"2022-08-13T12:15:14.698401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"most of the competitions were in the morning","metadata":{}},{"cell_type":"markdown","source":"# **Parameters of the race track** <a id=\"3\"></a>","metadata":{}},{"cell_type":"code","source":"nyra_2019_data","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:19.144233Z","iopub.execute_input":"2022-08-13T12:15:19.144996Z","iopub.status.idle":"2022-08-13T12:15:19.176400Z","shell.execute_reply.started":"2022-08-13T12:15:19.144946Z","shell.execute_reply":"2022-08-13T12:15:19.175191Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_data['distance_id'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:20.420359Z","iopub.execute_input":"2022-08-13T12:15:20.421415Z","iopub.status.idle":"2022-08-13T12:15:20.473193Z","shell.execute_reply.started":"2022-08-13T12:15:20.421370Z","shell.execute_reply":"2022-08-13T12:15:20.471752Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize = (12,6))\nsns.countplot(data = nyra_2019_data, x = 'distance_id')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:34.300189Z","iopub.execute_input":"2022-08-13T12:15:34.300626Z","iopub.status.idle":"2022-08-13T12:15:35.032747Z","shell.execute_reply.started":"2022-08-13T12:15:34.300583Z","shell.execute_reply":"2022-08-13T12:15:35.031411Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Most of all there were races for 6 and 7 furlongs","metadata":{}},{"cell_type":"code","source":"nyra_2019_data['track_condition'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:40.108506Z","iopub.execute_input":"2022-08-13T12:15:40.108930Z","iopub.status.idle":"2022-08-13T12:15:40.434208Z","shell.execute_reply.started":"2022-08-13T12:15:40.108896Z","shell.execute_reply":"2022-08-13T12:15:40.432764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = nyra_2019_data, x = 'track_condition')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:44.940434Z","iopub.execute_input":"2022-08-13T12:15:44.940866Z","iopub.status.idle":"2022-08-13T12:15:48.467701Z","shell.execute_reply.started":"2022-08-13T12:15:44.940827Z","shell.execute_reply":"2022-08-13T12:15:48.466502Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Usually the tracks were of the FT (Fast) and FM(Firm) categories","metadata":{}},{"cell_type":"code","source":"nyra_2019_data['course_type'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:53.920815Z","iopub.execute_input":"2022-08-13T12:15:53.921272Z","iopub.status.idle":"2022-08-13T12:15:54.148212Z","shell.execute_reply.started":"2022-08-13T12:15:53.921226Z","shell.execute_reply":"2022-08-13T12:15:54.146913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data = nyra_2019_data, x = 'course_type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T07:07:07.614506Z","iopub.execute_input":"2022-08-12T07:07:07.616209Z","iopub.status.idle":"2022-08-12T07:07:11.067056Z","shell.execute_reply.started":"2022-08-12T07:07:07.616158Z","shell.execute_reply":"2022-08-12T07:07:11.065603Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats_by_track_condition = nyra_2019_data.groupby(\"track_condition\")[\"purse\"].agg([\"mean\", \"median\", \"min\", \"max\", \"count\"])\nstats_by_track_course_type = nyra_2019_data.groupby(\"course_type\")[\"purse\"].agg([\"mean\", \"median\", \"min\", \"max\", \"count\"])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:15:59.488854Z","iopub.execute_input":"2022-08-13T12:15:59.489287Z","iopub.status.idle":"2022-08-13T12:16:00.936573Z","shell.execute_reply.started":"2022-08-13T12:15:59.489250Z","shell.execute_reply":"2022-08-13T12:16:00.935133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"stats_by_track_condition","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1 The highest average price of the Soft category.\n\n2 The Fast category had the most money\n\nI assume that this mode is the most common","metadata":{}},{"cell_type":"code","source":"stats_by_track_course_type","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:16:02.752300Z","iopub.execute_input":"2022-08-13T12:16:02.752762Z","iopub.status.idle":"2022-08-13T12:16:02.766159Z","shell.execute_reply.started":"2022-08-13T12:16:02.752725Z","shell.execute_reply":"2022-08-13T12:16:02.764927Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"1 The highest average price of the  I (Inner) category.\n\n2 The D(Dirt) category had the most money","metadata":{}},{"cell_type":"markdown","source":"# **Analysis hypotheses of categorical features** <a id=\"4\"></a>","metadata":{}},{"cell_type":"code","source":"from scipy import stats\nfrom statsmodels.stats.diagnostic import lilliefors","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:01:24.686194Z","iopub.execute_input":"2022-08-13T12:01:24.686631Z","iopub.status.idle":"2022-08-13T12:01:24.798439Z","shell.execute_reply.started":"2022-08-13T12:01:24.686594Z","shell.execute_reply":"2022-08-13T12:01:24.797061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_data","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:16:07.076190Z","iopub.execute_input":"2022-08-13T12:16:07.076648Z","iopub.status.idle":"2022-08-13T12:16:07.095040Z","shell.execute_reply.started":"2022-08-13T12:16:07.076607Z","shell.execute_reply":"2022-08-13T12:16:07.094065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_an_features = ['distance_id','run_up_distance','purse','post_time']\nsns.pairplot(data=nyra_2019_data, vars=num_an_features,\n             hue='track_id')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:16:12.192195Z","iopub.execute_input":"2022-08-13T12:16:12.192615Z","iopub.status.idle":"2022-08-13T12:16:18.695622Z","shell.execute_reply.started":"2022-08-13T12:16:12.192581Z","shell.execute_reply":"2022-08-13T12:16:18.693945Z"},"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"sns.pairplot(data=nyra_2019_data, vars=num_an_features,\n             hue='course_type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T12:19:18.045579Z","iopub.execute_input":"2022-08-12T12:19:18.045961Z","iopub.status.idle":"2022-08-12T13:11:05.896127Z","shell.execute_reply.started":"2022-08-12T12:19:18.045927Z","shell.execute_reply":"2022-08-12T13:11:05.894038Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"sns.pairplot(data=nyra_2019_data, vars=num_an_features,\n             hue='race_type')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:26:54.240633Z","iopub.execute_input":"2022-08-12T13:26:54.241608Z","iopub.status.idle":"2022-08-12T14:18:56.069239Z","shell.execute_reply.started":"2022-08-12T13:26:54.241567Z","shell.execute_reply":"2022-08-12T14:18:56.066871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"**ANOVA**","metadata":{}},{"cell_type":"markdown","source":"**1**","metadata":{}},{"cell_type":"code","source":"F, p = stats.f_oneway(nyra_2019_data[nyra_2019_data.track_condition=='FT '].purse,\n                      nyra_2019_data[nyra_2019_data.course_type=='D'].purse,\n                      nyra_2019_data[nyra_2019_data.race_type=='CLM'].purse)\n\nprint(F)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:22:19.245249Z","iopub.execute_input":"2022-08-12T13:22:19.245719Z","iopub.status.idle":"2022-08-12T13:22:21.523161Z","shell.execute_reply.started":"2022-08-12T13:22:19.245680Z","shell.execute_reply":"2022-08-12T13:22:21.521721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**2**","metadata":{}},{"cell_type":"code","source":"F, p = stats.f_oneway(nyra_2019_data[nyra_2019_data.track_condition=='FM '].purse,\n                      nyra_2019_data[nyra_2019_data.jockey=='Manuel Franco'].purse,\n                      nyra_2019_data[nyra_2019_data.track_id=='AQU'].purse)\nprint(F)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:24:24.437857Z","iopub.execute_input":"2022-08-12T13:24:24.438261Z","iopub.status.idle":"2022-08-12T13:24:26.161805Z","shell.execute_reply.started":"2022-08-12T13:24:24.438229Z","shell.execute_reply":"2022-08-12T13:24:26.160509Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**3**","metadata":{}},{"cell_type":"code","source":"F, p = stats.f_oneway(nyra_2019_data[nyra_2019_data.course_type=='D'].purse,\n                      nyra_2019_data[nyra_2019_data.track_condition=='FM '].purse,\n                      nyra_2019_data[nyra_2019_data.track_id=='AQU'].purse)\nprint(F)","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:24:32.268511Z","iopub.execute_input":"2022-08-12T13:24:32.269528Z","iopub.status.idle":"2022-08-12T13:24:34.510809Z","shell.execute_reply.started":"2022-08-12T13:24:32.269483Z","shell.execute_reply":"2022-08-12T13:24:34.509731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"strong correlation in the data","metadata":{}},{"cell_type":"markdown","source":"# **Analysis hypotheses of numetrical features** <a id=\"5\"></a>","metadata":{}},{"cell_type":"code","source":"for num_var in num_cols:\n    ax1, ax2 = plt.figure(figsize = (15,8)).subplots(1,2)\n    sns.histplot(data = nyra_2019_data, x = num_var, ax=ax1) \n    \n    sns.boxplot(data = nyra_2019_data, x = num_var, ax=ax2)\n    \n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2022-08-12T13:26:05.309943Z","iopub.execute_input":"2022-08-12T13:26:05.310632Z","iopub.status.idle":"2022-08-12T13:26:47.776028Z","shell.execute_reply.started":"2022-08-12T13:26:05.310568Z","shell.execute_reply":"2022-08-12T13:26:47.774901Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"two_histograms(nyra_2019_data['odds'],nyra_2019_data['trakus_index'])","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:00:15.470802Z","iopub.execute_input":"2022-08-13T12:00:15.471551Z","iopub.status.idle":"2022-08-13T12:00:52.041762Z","shell.execute_reply.started":"2022-08-13T12:00:15.471505Z","shell.execute_reply":"2022-08-13T12:00:52.040749Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Checking the normality of the distribution**","metadata":{}},{"cell_type":"code","source":"test_cond = stats.anderson(nyra_2019_data['run_up_distance'],'norm')[2]\ntest_cond","metadata":{"execution":{"iopub.status.busy":"2022-08-13T12:03:33.249535Z","iopub.execute_input":"2022-08-13T12:03:33.250091Z","iopub.status.idle":"2022-08-13T12:03:34.067197Z","shell.execute_reply.started":"2022-08-13T12:03:33.250050Z","shell.execute_reply":"2022-08-13T12:03:34.065922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"According to the reference for the normality of the distribution , the values must correspond to 15%, 10%, 5%, 2,5%, 1%. They came out from us.Then the hypothesis of the normality of the distribution cannot be rejected.","metadata":{}},{"cell_type":"markdown","source":"# **Map** <a id=\"6\"></a>","metadata":{}},{"cell_type":"code","source":"import plotly.express as px \nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots \nimport plotly.figure_factory as ff\nfrom plotly.offline import plot, iplot, init_notebook_mode\nimport plotly.graph_objs as go","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:02:08.795565Z","iopub.execute_input":"2022-08-28T07:02:08.795956Z","iopub.status.idle":"2022-08-28T07:02:11.212476Z","shell.execute_reply.started":"2022-08-28T07:02:08.795923Z","shell.execute_reply":"2022-08-28T07:02:11.211241Z"},"_kg_hide-input":true,"collapsed":true,"jupyter":{"outputs_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_data[\"track_id\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:05:15.884122Z","iopub.execute_input":"2022-08-28T07:05:15.884655Z","iopub.status.idle":"2022-08-28T07:05:16.196139Z","shell.execute_reply.started":"2022-08-28T07:05:15.884619Z","shell.execute_reply":"2022-08-28T07:05:16.194385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_data['race_date'] = pd.to_datetime(nyra_2019_data['race_date'])\nnyra_2019_data['day'], nyra_2019_data['month'] = nyra_2019_data['race_date'].dt.day, nyra_2019_data['race_date'].dt.month","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:02:19.901326Z","iopub.execute_input":"2022-08-28T07:02:19.901945Z","iopub.status.idle":"2022-08-28T07:02:21.930453Z","shell.execute_reply.started":"2022-08-28T07:02:19.901888Z","shell.execute_reply":"2022-08-28T07:02:21.928716Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for track in [\"AQU\",\"SAR\",\"BEL\"]:\n    nyra_tracking0 = nyra_2019_data[nyra_2019_data[\"track_id\"] == track]\n    nyra_tracking1 = nyra_2019_data[nyra_2019_data[\"month\"] == 11]\n\n    plt.style.use(\"dark_background\")\n    plt.figure(figsize = (16, 8))\n    sns.scatterplot(data = nyra_tracking1, x = \"longitude\", y = \"latitude\", hue = \"race_number\", palette = \"Paired\")\n\n    plt.title(f\"track id : {track}\")\n    plt.legend()\n","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:07:41.196012Z","iopub.execute_input":"2022-08-28T07:07:41.196665Z","iopub.status.idle":"2022-08-28T07:08:34.141773Z","shell.execute_reply.started":"2022-08-28T07:07:41.196619Z","shell.execute_reply":"2022-08-28T07:08:34.139652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **The importance of features** <a id=\"7\"></a>","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nfrom sklearn.tree import DecisionTreeClassifier\n\nfrom sklearn.tree import DecisionTreeRegressor","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:20:24.622924Z","iopub.execute_input":"2022-08-28T07:20:24.624401Z","iopub.status.idle":"2022-08-28T07:20:24.743496Z","shell.execute_reply.started":"2022-08-28T07:20:24.624331Z","shell.execute_reply":"2022-08-28T07:20:24.741954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_data","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:19:08.547385Z","iopub.execute_input":"2022-08-28T07:19:08.547804Z","iopub.status.idle":"2022-08-28T07:19:08.583093Z","shell.execute_reply.started":"2022-08-28T07:19:08.547771Z","shell.execute_reply":"2022-08-28T07:19:08.581564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nyra_2019_copy_data = nyra_2019_data.copy(deep= True)\nnyra_2019_copy_data = nyra_2019_copy_data.drop( columns = ['race_date','latitude','longitude','jockey','race_type'], axis = 1)\nnum_data = nyra_2019_copy_data.select_dtypes(np.number)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-08-28T07:37:03.891511Z","iopub.execute_input":"2022-08-28T07:37:03.892071Z","iopub.status.idle":"2022-08-28T07:37:05.559037Z","shell.execute_reply.started":"2022-08-28T07:37:03.892034Z","shell.execute_reply":"2022-08-28T07:37:05.557532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for num_label in num_data.columns:\n    print(f'======= START TRAIN {num_label} =======\\n')\n    train_data = nyra_2019_copy_data.copy(deep=True)\n    train_data = pd.get_dummies(train_data)\n    \n    model = DecisionTreeRegressor()\n    label = train_data.pop(num_label)\n    \n    X_train, X_test, y_train, y_test = train_test_split(train_data,label, test_size = 0.25)\n    model.fit(X_train,y_train)\n    \n    feature_importance = pd.DataFrame({'feature':train_data.columns, 'importance': model.feature_importances_}).sort_values('importance')\n    print(feature_importance)\n    print('\\n')","metadata":{"execution":{"iopub.status.busy":"2022-08-28T07:40:27.931054Z","iopub.execute_input":"2022-08-28T07:40:27.931495Z","iopub.status.idle":"2022-08-28T07:50:50.762109Z","shell.execute_reply.started":"2022-08-28T07:40:27.931460Z","shell.execute_reply":"2022-08-28T07:50:50.760167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ai = anai.run(filepath = '../input/big-data-derby-2022/nyra_start_table.csv', target = 'odds', suppress_task_detection=True, task = 'regression',  predictor = ['ada', 'cat', 'knn', 'lgbm'])","metadata":{"execution":{"iopub.status.busy":"2022-08-29T05:26:50.322239Z","iopub.execute_input":"2022-08-29T05:26:50.325827Z","iopub.status.idle":"2022-08-29T05:27:43.928011Z","shell.execute_reply.started":"2022-08-29T05:26:50.325742Z","shell.execute_reply":"2022-08-29T05:27:43.922100Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ai.explain('shap')","metadata":{"execution":{"iopub.status.busy":"2022-08-29T05:34:06.244491Z","iopub.execute_input":"2022-08-29T05:34:06.245073Z","iopub.status.idle":"2022-08-29T05:34:56.165831Z","shell.execute_reply.started":"2022-08-29T05:34:06.245032Z","shell.execute_reply":"2022-08-29T05:34:56.162999Z"},"trusted":true},"execution_count":null,"outputs":[]}]}