{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Overview\n* EDA with Plotly was found to have a high cardinality of horse riders.\n* **Starter with DBScan**\n    * By using DBScan to classify jockey, I am able to classify riders into 102 classes with the least number of rides as outliers. \n    * 102 classes can be classified by the number of classes in which the jockey belongs to more than one class. \n    * The jockey with the highest number of appearances could be easily classified by DBScan.\n* **Tracking**\n    * Clustering using GMM was performed on horse location data, but only racetrack-level separation was achieved.\n    * Currently, I am considering splitting the data into racetrack and course levels before categorizing the individual data.\n    \n# Motivation\n* I have never used Plotly before, so I tried it. It is surprisingly easy to draw beautiful graphs.\n\n>Thank you @marcociav\n>[plotly tutorial be a visualization grandmaster](https://www.kaggle.com/code/chazzer/plotly-tutorial-be-a-visualization-grandmaster)\n\n* The horse's placements were made public. With the current data alone, we can only sort out the jockeys, but we believe that horses are living organisms and need to be classified.\n\n>Thank you @`Mark Green`\n[Estimating clustering via convex hulls](https://www.kaggle.com/code/themarkgreen/estimating-clustering-via-convex-hulls)\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19"}},{"cell_type":"markdown","source":"# Import Liblary","metadata":{}},{"cell_type":"code","source":"!pip install utm\nimport utm\nimport datetime\n\nimport os\nimport re\nimport warnings\n\n# for analyze data\nimport numpy as np \nimport pandas as pd \nfrom shapely.geometry import Point\nfrom shapely.geometry.polygon import Polygon\nfrom tqdm.auto import tqdm\ntqdm.pandas()\n\n# for visualization\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom matplotlib import mlab\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nfrom plotly import subplots\n\n# for demension reduction\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import PCA\n\n# for clusterling\nfrom sklearn.cluster import KMeans, DBSCAN\nfrom sklearn.mixture import GaussianMixture, BayesianGaussianMixture\n\nwarnings.simplefilter('ignore')\n\n#!pip install pymap3d\n#import pymap3d as pm\nsns.set_style(\"darkgrid\")","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:45:40.858871Z","iopub.execute_input":"2022-09-19T22:45:40.859318Z","iopub.status.idle":"2022-09-19T22:45:58.165981Z","shell.execute_reply.started":"2022-09-19T22:45:40.859229Z","shell.execute_reply":"2022-09-19T22:45:58.164754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Extract Data","metadata":{}},{"cell_type":"code","source":"dtype_dict = {\"track_id\": object ,\n              \"race_date\" :object, #will be read with \"parse_dates\" parameter\n              \"race_number\" : np.int8, \n              \"program_number\" : object,\n              \"trakus_index\" : np.int16,  \n              \"latitude\" : np.float64,\n              \"longitude\" : np.float64,\n              \"distance_id\" : np.int16,  \n              \"course_type\" : object, \n              \"track_condition\" : object,\n              \"run_up_distance\" : np.int8,  \n              \"race_type\" : object, \n              \"purse\" : np.int32,  \n              \"post_time\" : np.int16,  \n              \"weight_carried\" : np.int16,  \n              \"jockey\" : object, \n              \"odds\" : np.int16,\n              \"position_at_finish\":np.int8\n             }\nALL_PATH = '/kaggle/input/big-data-derby-2022/nyra_2019_complete.csv'\nall_df = pd.read_csv(ALL_PATH, header=None,dtype=dtype_dict,parse_dates=[1])\nall_df.columns= dtype_dict.keys()\n\nall_df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:45:58.167637Z","iopub.execute_input":"2022-09-19T22:45:58.167938Z","iopub.status.idle":"2022-09-19T22:46:16.078338Z","shell.execute_reply.started":"2022-09-19T22:45:58.167910Z","shell.execute_reply":"2022-09-19T22:46:16.077047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create key for each race\nall_df['race_id'] = all_df['track_id'] +'-'+ all_df['race_date'].astype(str) \\\n                    + '-'+ all_df['race_number'].astype(str)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:46:16.079959Z","iopub.execute_input":"2022-09-19T22:46:16.080706Z","iopub.status.idle":"2022-09-19T22:46:52.101175Z","shell.execute_reply.started":"2022-09-19T22:46:16.080663Z","shell.execute_reply":"2022-09-19T22:46:52.099113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"HORSE_PATH='../input/big-data-derby-2022-global-horse-ids-and-places/horse_ids.csv'\nhorse_df = pd.read_csv(HORSE_PATH)\nhorse_lst = ['track_id', 'race_date', 'race', 'program_number','horse_id']\nhorse_df = horse_df[horse_lst].rename(columns={'race':'race_number'})\nhorse_df.race_date = pd.to_datetime(horse_df.race_date)\nhorse_lst = list(horse_df.columns)\n\nhorse_df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:46:52.104934Z","iopub.execute_input":"2022-09-19T22:46:52.105552Z","iopub.status.idle":"2022-09-19T22:46:52.161589Z","shell.execute_reply.started":"2022-09-19T22:46:52.105517Z","shell.execute_reply":"2022-09-19T22:46:52.160780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# drop blank in `program_number` column to merge dataframe\nall_df.program_number=all_df.program_number.apply(lambda x: str(x).replace(' ',''))\nhorse_df.program_number=horse_df.program_number.apply(lambda x: str(x).replace(' ',''))\n\n# merge dataframe\nall_df=all_df.merge(horse_df,on=horse_lst[:-1])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:46:52.162927Z","iopub.execute_input":"2022-09-19T22:46:52.163510Z","iopub.status.idle":"2022-09-19T22:47:01.280360Z","shell.execute_reply.started":"2022-09-19T22:46:52.163478Z","shell.execute_reply":"2022-09-19T22:47:01.279283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start_dict = {\"race_id\":object,\n              \"track_id\": object ,\n              \"race_date\" :object,\n              \"race_number\" : np.int8, \n              \"program_number\":np.int8,\n              \"weight_carried\" : np.int16,  \n              \"jockey\" : object, \n              \"odds\" : np.int16,\n              \"position_at_finish\":np.int8,\n              \"horse_id\":np.int8\n             }\nstart_df = all_df[start_dict.keys()].drop_duplicates()\nstart_df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:01.282431Z","iopub.execute_input":"2022-09-19T22:47:01.283290Z","iopub.status.idle":"2022-09-19T22:47:06.101783Z","shell.execute_reply.started":"2022-09-19T22:47:01.283246Z","shell.execute_reply":"2022-09-19T22:47:06.100436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"race_dict = {\"race_id\":object,\n              \"track_id\": object ,\n              \"race_date\" :object,\n              \"race_number\" : np.int8, \n              \"distance_id\" : np.int16,  \n              \"course_type\" : object, \n              \"track_condition\" : object,\n              \"run_up_distance\" : np.int8,  \n              \"race_type\" : object,\n              \"purse\" : np.int32,  \n              \"post_time\" : np.int16,  \n             }\nrace_df = all_df[race_dict.keys()].drop_duplicates()\nrace_df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:06.103525Z","iopub.execute_input":"2022-09-19T22:47:06.104033Z","iopub.status.idle":"2022-09-19T22:47:09.258943Z","shell.execute_reply.started":"2022-09-19T22:47:06.103980Z","shell.execute_reply":"2022-09-19T22:47:09.257760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracking_dict = {\"race_id\":object,\n              \"track_id\": object ,\n              \"race_date\" :object, #will be read with \"parse_dates\" parameter\n              \"race_number\" : np.int8, \n              \"program_number\" : object,\n              \"trakus_index\" : np.int16,  \n              \"latitude\" : np.float64,\n              \"longitude\" : np.float64,\n             }\ntracking_df = all_df[tracking_dict.keys()].drop_duplicates()\ntracking_df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:09.260422Z","iopub.execute_input":"2022-09-19T22:47:09.261503Z","iopub.status.idle":"2022-09-19T22:47:15.768867Z","shell.execute_reply.started":"2022-09-19T22:47:09.261460Z","shell.execute_reply":"2022-09-19T22:47:15.767696Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"analyze_dict = {\"race_id\":object,\n              \"track_id\": object ,\n              \"race_date\" :object, #will be read with \"parse_dates\" parameter\n              \"race_number\" : np.int8, \n              \"program_number\" : object,\n              \"distance_id\" : np.int16,  \n              \"course_type\" : object, \n              \"track_condition\" : object,\n              \"run_up_distance\" : np.int8,  \n              \"race_type\" : object, \n              \"purse\" : np.int32,  \n              \"post_time\" : np.int16,  \n              \"weight_carried\" : np.int16,  \n              \"jockey\" : object, \n              \"odds\" : np.int16,\n              \"horse_id\":np.int8,\n              \"position_at_finish\":np.int8\n             }\n\nanalyze_df = all_df[analyze_dict.keys()].drop_duplicates()\nanalyze_df.info()","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:15.770062Z","iopub.execute_input":"2022-09-19T22:47:15.770890Z","iopub.status.idle":"2022-09-19T22:47:20.695417Z","shell.execute_reply.started":"2022-09-19T22:47:15.770857Z","shell.execute_reply":"2022-09-19T22:47:20.694265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA - Race","metadata":{}},{"cell_type":"markdown","source":"## Uni-variable","metadata":{}},{"cell_type":"code","source":"def agg_uni_count_ratio(df,col,count_,ratio_):\n    #　Get Count\n    count_df = df[col].value_counts().to_frame().reset_index()\n    count_df = count_df.rename(columns={'index':col,col:count_})\n    \n    # Get Percentage\n    ratio_df = 100*df[col].value_counts()/df.shape[0]\n    ratio_df = ratio_df.to_frame().reset_index()\n    ratio_df = ratio_df.rename(columns={'index':col,col:ratio_})\n    \n    return count_df,ratio_df\n\ndef uni_val_bar(df,target_col,count_='Count',ratio_='Ratio'):\n    # Execute functions to aggregate data\n    # target_col:\n    # count_:　Name for count in chart\n    # ratio_: Name for ratio in chart\n    count_df,ratio_df = agg_uni_count_ratio(df,target_col,count_,ratio_)\n    fig= subplots.make_subplots(\n        # \n        rows=1, cols=2,\n        subplot_titles=[target_col+':'+'Value '+count_, \n                        target_col+':'+'Value '+ratio_]\n        ,x_title=target_col)\n    \n    fig.add_trace(go.Bar(x=count_df[target_col], y=count_df.Count\n                         , name=count_),\n              row=1, col=1)\n    fig.add_trace(go.Bar(x=ratio_df[target_col], y=ratio_df.Ratio\n                         , name=ratio_),\n              row=1, col=2)\n    return fig\n\n\ndef uni_val_histgram(df,target_col_):\n    # Assume `from matplotlib import pyplot as plt`\n    # below is same as plt.subplots(1,2)\n    fig= subplots.make_subplots(\n        rows=1, cols=2,\n        # Sub-Title and x_label in matplotlib is set in this function\n        subplot_titles=[target_col_+':'+'Value Count', \n                        target_col_+':'+'Value Ratio']\n        ,x_title=target_col_)\n    # plt.subplot(1,2,1)\n    fig.add_trace(go.Histogram(x=df[target_col_],name='Count'),\n              row=1, col=1)\n    # plt.subplot(1,2,2)\n    fig.add_trace(go.Histogram(x=df[target_col_], histnorm='probability',name='Ratio'),\n              row=1, col=2)\n    \n    return fig\n\ndef uni_val_heatmap(df,target_col_):\n    count_df = df[target_col_].value_counts()\n    count_df = count_df.to_frame().reset_index()\n    count_df = count_df.rename(columns={'index':target_col_,target_col_:'Count'})\n    count_df[target_col_] = count_df[target_col_].dt.week\n    \n    # Define heatmap\n    fig = px.density_heatmap(count_df,x=target_col_, y='Count')\n\n    return fig","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:20.699774Z","iopub.execute_input":"2022-09-19T22:47:20.700131Z","iopub.status.idle":"2022-09-19T22:47:20.714978Z","shell.execute_reply.started":"2022-09-19T22:47:20.700099Z","shell.execute_reply":"2022-09-19T22:47:20.713497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_cat_col_lst = []\ntarget_num_col_lst = []\ntarget_other_col_lst = []\n\nfor target_col in race_df.columns:\n    if (target_col not in ['race_id']):\n        if race_df[target_col].dtype=='object':\n            target_cat_col_lst.append(target_col)\n        elif ('int' in str(race_df[target_col].dtype)) or ('float' in str(race_df[target_col].dtype)):\n            \n            target_num_col_lst.append(target_col)\n        else:\n            target_other_col_lst.append(target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:20.716257Z","iopub.execute_input":"2022-09-19T22:47:20.716594Z","iopub.status.idle":"2022-09-19T22:47:20.731820Z","shell.execute_reply.started":"2022-09-19T22:47:20.716564Z","shell.execute_reply":"2022-09-19T22:47:20.730719Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_col = target_cat_col_lst[0]\nuni_val_bar(race_df,target_col,'Count','Ratio')","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:20.733221Z","iopub.execute_input":"2022-09-19T22:47:20.733691Z","iopub.status.idle":"2022-09-19T22:47:21.012840Z","shell.execute_reply.started":"2022-09-19T22:47:20.733660Z","shell.execute_reply":"2022-09-19T22:47:21.011844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_bar(race_df,target_cat_col_lst[1],'Count','Ratio')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.013947Z","iopub.execute_input":"2022-09-19T22:47:21.016605Z","iopub.status.idle":"2022-09-19T22:47:21.059588Z","shell.execute_reply.started":"2022-09-19T22:47:21.016574Z","shell.execute_reply":"2022-09-19T22:47:21.058190Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_bar(race_df,target_cat_col_lst[2],'Count','Ratio')","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.060957Z","iopub.execute_input":"2022-09-19T22:47:21.061394Z","iopub.status.idle":"2022-09-19T22:47:21.106149Z","shell.execute_reply.started":"2022-09-19T22:47:21.061360Z","shell.execute_reply":"2022-09-19T22:47:21.105032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_histgram(race_df,target_num_col_lst[0])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.107544Z","iopub.execute_input":"2022-09-19T22:47:21.107956Z","iopub.status.idle":"2022-09-19T22:47:21.154958Z","shell.execute_reply.started":"2022-09-19T22:47:21.107926Z","shell.execute_reply":"2022-09-19T22:47:21.153951Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_histgram(race_df,target_num_col_lst[1])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.156038Z","iopub.execute_input":"2022-09-19T22:47:21.157020Z","iopub.status.idle":"2022-09-19T22:47:21.428905Z","shell.execute_reply.started":"2022-09-19T22:47:21.156954Z","shell.execute_reply":"2022-09-19T22:47:21.428145Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_histgram(race_df,target_num_col_lst[2])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:21.429918Z","iopub.execute_input":"2022-09-19T22:47:21.430505Z","iopub.status.idle":"2022-09-19T22:47:21.465800Z","shell.execute_reply.started":"2022-09-19T22:47:21.430475Z","shell.execute_reply":"2022-09-19T22:47:21.464398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_histgram(race_df,target_num_col_lst[3])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.467428Z","iopub.execute_input":"2022-09-19T22:47:21.468175Z","iopub.status.idle":"2022-09-19T22:47:21.509403Z","shell.execute_reply.started":"2022-09-19T22:47:21.468132Z","shell.execute_reply":"2022-09-19T22:47:21.508338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_histgram(race_df,target_num_col_lst[4])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.510862Z","iopub.execute_input":"2022-09-19T22:47:21.511517Z","iopub.status.idle":"2022-09-19T22:47:21.550617Z","shell.execute_reply.started":"2022-09-19T22:47:21.511475Z","shell.execute_reply":"2022-09-19T22:47:21.549502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"uni_val_heatmap(race_df,target_other_col_lst[0])","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:21.552849Z","iopub.execute_input":"2022-09-19T22:47:21.553516Z","iopub.status.idle":"2022-09-19T22:47:22.275780Z","shell.execute_reply.started":"2022-09-19T22:47:21.553478Z","shell.execute_reply":"2022-09-19T22:47:22.274721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Bi-variable","metadata":{}},{"cell_type":"code","source":"def agg_bi_count_ratio(df,subset_col_,target_col_):\n    total_df = df.groupby([subset_col_])['race_id'].count().to_frame().reset_index()\n    total_df = total_df.rename(columns={'race_id':'Total Count'})\n    \n    subset_df = df.groupby([subset_col_,target_col_])['race_id'].count().to_frame().reset_index()\n    subset_df = subset_df.rename(columns={'race_id':'Each Count'})\n    \n    subset_df = subset_df.merge(total_df,on=[subset_col_])\n\n    return subset_df\n\ndef bi_val_bar(df,subset_col_,target_col_):\n    # Apply function to get aggregate value\n    bi_val_df = agg_bi_count_ratio(df,subset_col_,target_col_)\n    \n    # create bar chart with hue\n    # hue is  handled with `color` parameter\n    fig = px.bar(bi_val_df,x=target_col_, y='Each Count'\n                 ,color=subset_col_)\n    \n    return fig\n\n\ndef bi_val_histgram(df,subset_col_,target_col_):\n    fig= subplots.make_subplots(\n        rows=1, cols=2,\n        subplot_titles=[target_col+':'+'Value Count', \n                        target_col+':'+'Value Ratio']\n        ,x_title=target_col)\n    \n    for col_name in df[subset_col_].unique():\n        # x should be got by pandas dataseries\n        # name is same as hue in matplotlib\n        fig.add_trace(go.Histogram(x=df[df[subset_col_]==col_name][target_col_]\n                                   ,name=col_name + ':Count'),\n              row=1, col=1)\n        fig.add_trace(go.Histogram(x=df[df[subset_col_]==col_name][target_col_]\n                                   , histnorm='probability'\n                                   ,name =col_name + ':Ratio'),\n              row=1, col=2)\n    # title and label is set in function\n    fig.update_layout(\n        title_text= target_col_ +' ', # title of plot\n        xaxis_title_text=target_col_, # xaxis label\n        bargap=0.2, # gap between bars of adjacent location coordinates\n        bargroupgap=0.1 # gap between bars of the same location coordinates)\n    )\n    return fig\n\ndef bi_val_boxplot(df,subset_col_,target_col_):\n    fig = px.box(df, x=subset_col_, y=target_col_)\n    \n    return fig\n\n\ndef bi_val_heatmap(df,facet_col_,target_col_):\n    bi_val_df = agg_bi_count_ratio(df,facet_col_,target_col_)\n    \n    if target_col_ =='race_date':\n        bi_val_df[target_col_] = bi_val_df[target_col_].dt.week\n    \n    #  Faset is defined in plot function\n    fig =  px.density_heatmap(bi_val_df,x=target_col_, y='Each Count'\n                              ,facet_col=facet_col_)\n\n    return fig","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.277908Z","iopub.execute_input":"2022-09-19T22:47:22.278429Z","iopub.status.idle":"2022-09-19T22:47:22.291529Z","shell.execute_reply.started":"2022-09-19T22:47:22.278398Z","shell.execute_reply":"2022-09-19T22:47:22.290547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_cat_col_lst = []\ntarget_num_col_lst = []\ntarget_other_col_lst = []\n\nfor target_col in race_df.columns:\n    if (target_col not in ['race_id',subset_col]):\n        if race_df[target_col].dtype=='object':\n            target_cat_col_lst.append(target_col)\n        elif ('int' in str(race_df[target_col].dtype)) or ('float' in str(race_df[target_col].dtype)):\n            \n            target_num_col_lst.append(target_col)\n        else:\n            target_other_col_lst.append(target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.294899Z","iopub.execute_input":"2022-09-19T22:47:22.295361Z","iopub.status.idle":"2022-09-19T22:47:22.306554Z","shell.execute_reply.started":"2022-09-19T22:47:22.295330Z","shell.execute_reply":"2022-09-19T22:47:22.305500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Category","metadata":{}},{"cell_type":"code","source":"target_col=target_cat_col_lst[0]\nsubset_col ='track_id'\nbi_val_bar(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.308145Z","iopub.execute_input":"2022-09-19T22:47:22.308572Z","iopub.status.idle":"2022-09-19T22:47:22.405669Z","shell.execute_reply.started":"2022-09-19T22:47:22.308531Z","shell.execute_reply":"2022-09-19T22:47:22.404530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_cat_col_lst[1]\nbi_val_bar(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.406894Z","iopub.execute_input":"2022-09-19T22:47:22.407834Z","iopub.status.idle":"2022-09-19T22:47:22.486147Z","shell.execute_reply.started":"2022-09-19T22:47:22.407789Z","shell.execute_reply":"2022-09-19T22:47:22.485053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_cat_col_lst[2]\nbi_val_bar(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.487793Z","iopub.execute_input":"2022-09-19T22:47:22.488137Z","iopub.status.idle":"2022-09-19T22:47:22.565594Z","shell.execute_reply.started":"2022-09-19T22:47:22.488106Z","shell.execute_reply":"2022-09-19T22:47:22.564821Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number","metadata":{}},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_num_col_lst[0]\nbi_val_boxplot(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.567040Z","iopub.execute_input":"2022-09-19T22:47:22.567607Z","iopub.status.idle":"2022-09-19T22:47:22.654901Z","shell.execute_reply.started":"2022-09-19T22:47:22.567575Z","shell.execute_reply":"2022-09-19T22:47:22.653799Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col = target_num_col_lst[1]\nbi_val_boxplot(race_df,subset_col,target_col)\n","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.656345Z","iopub.execute_input":"2022-09-19T22:47:22.656677Z","iopub.status.idle":"2022-09-19T22:47:22.721730Z","shell.execute_reply.started":"2022-09-19T22:47:22.656647Z","shell.execute_reply":"2022-09-19T22:47:22.720915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_num_col_lst[2]\nbi_val_boxplot(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.728876Z","iopub.execute_input":"2022-09-19T22:47:22.729505Z","iopub.status.idle":"2022-09-19T22:47:22.794683Z","shell.execute_reply.started":"2022-09-19T22:47:22.729467Z","shell.execute_reply":"2022-09-19T22:47:22.793566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_num_col_lst[3]\nbi_val_boxplot(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.796502Z","iopub.execute_input":"2022-09-19T22:47:22.797196Z","iopub.status.idle":"2022-09-19T22:47:22.864268Z","shell.execute_reply.started":"2022-09-19T22:47:22.797151Z","shell.execute_reply":"2022-09-19T22:47:22.863039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_num_col_lst[4]\nbi_val_boxplot(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.866047Z","iopub.execute_input":"2022-09-19T22:47:22.867313Z","iopub.status.idle":"2022-09-19T22:47:22.934751Z","shell.execute_reply.started":"2022-09-19T22:47:22.867279Z","shell.execute_reply":"2022-09-19T22:47:22.933456Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Datetime","metadata":{}},{"cell_type":"code","source":"subset_col ='track_id'\ntarget_col=target_other_col_lst[0]\nbi_val_heatmap(race_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:22.936618Z","iopub.execute_input":"2022-09-19T22:47:22.937590Z","iopub.status.idle":"2022-09-19T22:47:23.077689Z","shell.execute_reply.started":"2022-09-19T22:47:22.937543Z","shell.execute_reply":"2022-09-19T22:47:23.076742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"code","source":"def agg_facet_count(df,facet_col_,subset_col_,target_col_):\n    facet_df = df.groupby([facet_col_,subset_col_,target_col_])['race_id'].count().to_frame().reset_index()\n\n    return facet_df\n\ndef facet_bar(df,facet_col_,subset_col_,target_col_):\n    facet_df = agg_facet_count(df,facet_col_,subset_col_,target_col_)\n    facet_df = facet_df.rename(columns={'race_id':'Each Count'})\n    \n    # Facet bar chart with hue\n    fig = px.bar(facet_df,x=target_col_, y='Each Count'\n                 ,color=subset_col_\n                 ,facet_col=facet_col_)\n    return fig\n\ndef facet_histogram(df,facet_col_,subset_col_,target_col_):\n    # Facet histogram with hue\n    fig = px.histogram(df,x=target_col_\n                 ,color=subset_col_\n                 ,facet_col=facet_col_)\n    return fig\n\ndef facet_boxplot(df,facet_col_,subset_col_,target_col_):\n    fig = px.box(df, x=subset_col_, y=target_col_\n                 ,facet_col = facet_col_)\n    \n    return fig\n\ndef facet_scatter(df,facet_col_,subset_col_,target_col_,val_):\n    # Facet scatter plot with hue\n    fig = px.scatter(df,x=target_col_,y=val_\n                 ,color=subset_col_\n                 ,facet_col=facet_col_)\n    return fig","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:23.079414Z","iopub.execute_input":"2022-09-19T22:47:23.079743Z","iopub.status.idle":"2022-09-19T22:47:23.091074Z","shell.execute_reply.started":"2022-09-19T22:47:23.079714Z","shell.execute_reply":"2022-09-19T22:47:23.090051Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_cat_col_lst = []\ntarget_num_col_lst = []\ntarget_other_col_lst = []\n\nfor target_col in race_df.columns:\n    if (target_col not in ['race_id',subset_col,facet_col]):\n        if race_df[target_col].dtype=='object':\n            target_cat_col_lst.append(target_col)\n        elif ('int' in str(race_df[target_col].dtype)) or ('float' in str(race_df[target_col].dtype)):\n            \n            target_num_col_lst.append(target_col)\n        else:\n            target_other_col_lst.append(target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:23.092759Z","iopub.execute_input":"2022-09-19T22:47:23.094023Z","iopub.status.idle":"2022-09-19T22:47:23.104565Z","shell.execute_reply.started":"2022-09-19T22:47:23.093964Z","shell.execute_reply":"2022-09-19T22:47:23.103384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Category","metadata":{}},{"cell_type":"code","source":"target_col=target_cat_col_lst[0]\nfacet_bar(race_df,facet_col,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:23.107211Z","iopub.execute_input":"2022-09-19T22:47:23.107593Z","iopub.status.idle":"2022-09-19T22:47:23.270067Z","shell.execute_reply.started":"2022-09-19T22:47:23.107555Z","shell.execute_reply":"2022-09-19T22:47:23.269172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_col=target_cat_col_lst[1]\nfacet_bar(race_df,facet_col,subset_col,target_col)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:23.271163Z","iopub.execute_input":"2022-09-19T22:47:23.272173Z","iopub.status.idle":"2022-09-19T22:47:23.443144Z","shell.execute_reply.started":"2022-09-19T22:47:23.272135Z","shell.execute_reply":"2022-09-19T22:47:23.442066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number","metadata":{}},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_col=target_num_col_lst[0]\nfacet_boxplot(race_df,facet_col,subset_col,target_col)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:23.444539Z","iopub.execute_input":"2022-09-19T22:47:23.445342Z","iopub.status.idle":"2022-09-19T22:47:23.552373Z","shell.execute_reply.started":"2022-09-19T22:47:23.445295Z","shell.execute_reply":"2022-09-19T22:47:23.551195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_col = target_num_col_lst[1]\nfacet_boxplot(race_df,facet_col,subset_col,target_col)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:23.554389Z","iopub.execute_input":"2022-09-19T22:47:23.554816Z","iopub.status.idle":"2022-09-19T22:47:23.660896Z","shell.execute_reply.started":"2022-09-19T22:47:23.554775Z","shell.execute_reply":"2022-09-19T22:47:23.659717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_col = target_num_col_lst[2]\nfacet_boxplot(race_df,facet_col,subset_col,target_col)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:23.662551Z","iopub.execute_input":"2022-09-19T22:47:23.662982Z","iopub.status.idle":"2022-09-19T22:47:23.777422Z","shell.execute_reply.started":"2022-09-19T22:47:23.662941Z","shell.execute_reply":"2022-09-19T22:47:23.776226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_col = target_num_col_lst[3]\nfacet_boxplot(race_df,facet_col,subset_col,target_col)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:23.779072Z","iopub.execute_input":"2022-09-19T22:47:23.779499Z","iopub.status.idle":"2022-09-19T22:47:23.888576Z","shell.execute_reply.started":"2022-09-19T22:47:23.779457Z","shell.execute_reply":"2022-09-19T22:47:23.886020Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_col = target_num_col_lst[4]\nfacet_boxplot(race_df,facet_col,subset_col,target_col)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:23.890746Z","iopub.execute_input":"2022-09-19T22:47:23.891218Z","iopub.status.idle":"2022-09-19T22:47:23.995872Z","shell.execute_reply.started":"2022-09-19T22:47:23.891173Z","shell.execute_reply":"2022-09-19T22:47:23.994759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# EDA -Start","metadata":{}},{"cell_type":"code","source":"target_cat_col_lst = []\ntarget_num_col_lst = []\ntarget_other_col_lst = []\n\nfor target_col in start_df.columns:\n    if (target_col not in ['race_id']):\n        if start_df[target_col].dtype=='object':\n            target_cat_col_lst.append(target_col)\n        elif ('int' in str(start_df[target_col].dtype)) or ('float' in str(start_df[target_col].dtype)):\n            \n            target_num_col_lst.append(target_col)\n        else:\n            target_other_col_lst.append(target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:23.997383Z","iopub.execute_input":"2022-09-19T22:47:23.997724Z","iopub.status.idle":"2022-09-19T22:47:24.004555Z","shell.execute_reply.started":"2022-09-19T22:47:23.997694Z","shell.execute_reply":"2022-09-19T22:47:24.003434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Category","metadata":{}},{"cell_type":"code","source":"target_col=target_cat_col_lst[2]\nuni_val_bar(start_df,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.005962Z","iopub.execute_input":"2022-09-19T22:47:24.006946Z","iopub.status.idle":"2022-09-19T22:47:24.054115Z","shell.execute_reply.started":"2022-09-19T22:47:24.006904Z","shell.execute_reply":"2022-09-19T22:47:24.053079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Number","metadata":{}},{"cell_type":"code","source":"target_col=target_num_col_lst[1]\nuni_val_histgram(start_df,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.056053Z","iopub.execute_input":"2022-09-19T22:47:24.056483Z","iopub.status.idle":"2022-09-19T22:47:24.098328Z","shell.execute_reply.started":"2022-09-19T22:47:24.056440Z","shell.execute_reply":"2022-09-19T22:47:24.097021Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_col=target_num_col_lst[2]\nuni_val_histgram(start_df,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.100086Z","iopub.execute_input":"2022-09-19T22:47:24.100492Z","iopub.status.idle":"2022-09-19T22:47:24.140789Z","shell.execute_reply.started":"2022-09-19T22:47:24.100454Z","shell.execute_reply":"2022-09-19T22:47:24.139627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_col=target_num_col_lst[3]\nuni_val_histgram(start_df,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.142440Z","iopub.execute_input":"2022-09-19T22:47:24.143142Z","iopub.status.idle":"2022-09-19T22:47:24.185083Z","shell.execute_reply.started":"2022-09-19T22:47:24.143098Z","shell.execute_reply":"2022-09-19T22:47:24.184026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_col=target_num_col_lst[4]\nuni_val_histgram(start_df,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.186770Z","iopub.execute_input":"2022-09-19T22:47:24.187107Z","iopub.status.idle":"2022-09-19T22:47:24.228537Z","shell.execute_reply.started":"2022-09-19T22:47:24.187078Z","shell.execute_reply":"2022-09-19T22:47:24.227424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col=target_cat_col_lst[0]\ntarget_col=target_num_col_lst[1]\nbi_val_boxplot(start_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.230186Z","iopub.execute_input":"2022-09-19T22:47:24.230851Z","iopub.status.idle":"2022-09-19T22:47:24.344288Z","shell.execute_reply.started":"2022-09-19T22:47:24.230812Z","shell.execute_reply":"2022-09-19T22:47:24.343232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col=target_cat_col_lst[0]\ntarget_col=target_num_col_lst[2]\nbi_val_boxplot(start_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.345779Z","iopub.execute_input":"2022-09-19T22:47:24.346157Z","iopub.status.idle":"2022-09-19T22:47:24.460612Z","shell.execute_reply.started":"2022-09-19T22:47:24.346125Z","shell.execute_reply":"2022-09-19T22:47:24.459322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col=target_cat_col_lst[0]\ntarget_col=target_num_col_lst[3]\nbi_val_boxplot(start_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.462262Z","iopub.execute_input":"2022-09-19T22:47:24.463100Z","iopub.status.idle":"2022-09-19T22:47:24.578793Z","shell.execute_reply.started":"2022-09-19T22:47:24.463051Z","shell.execute_reply":"2022-09-19T22:47:24.577705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"subset_col=target_cat_col_lst[0]\ntarget_col=target_num_col_lst[4]\nbi_val_boxplot(start_df,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.580217Z","iopub.execute_input":"2022-09-19T22:47:24.581210Z","iopub.status.idle":"2022-09-19T22:47:24.697921Z","shell.execute_reply.started":"2022-09-19T22:47:24.581174Z","shell.execute_reply":"2022-09-19T22:47:24.696805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col=target_cat_col_lst[0]\nsubset_col=target_cat_col_lst[2]\ntarget_col=target_num_col_lst[4]\nfacet_histogram(start_df,facet_col,subset_col,target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:24.699732Z","iopub.execute_input":"2022-09-19T22:47:24.700246Z","iopub.status.idle":"2022-09-19T22:47:26.025835Z","shell.execute_reply.started":"2022-09-19T22:47:24.700198Z","shell.execute_reply":"2022-09-19T22:47:26.024957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# EDA-Combined","metadata":{}},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col ='course_type'\ntarget_cat_col_lst = []\ntarget_num_col_lst = []\ntarget_other_col_lst = []\n\nfor target_col in analyze_df.columns:\n    if analyze_df[target_col].dtype=='object':\n        target_cat_col_lst.append(target_col)\n    elif ('int' in str(analyze_df[target_col].dtype)) or ('float' in str(analyze_df[target_col].dtype)):\n\n        target_num_col_lst.append(target_col)\n    else:\n        target_other_col_lst.append(target_col)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:47:26.027213Z","iopub.execute_input":"2022-09-19T22:47:26.027760Z","iopub.status.idle":"2022-09-19T22:47:26.034329Z","shell.execute_reply.started":"2022-09-19T22:47:26.027728Z","shell.execute_reply":"2022-09-19T22:47:26.033293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"facet_col ='track_id'\nsubset_col = 'position_at_finish'\nfacet_scatter(analyze_df,facet_col,subset_col,'odds','purse')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:26.036441Z","iopub.execute_input":"2022-09-19T22:47:26.036858Z","iopub.status.idle":"2022-09-19T22:47:26.172991Z","shell.execute_reply.started":"2022-09-19T22:47:26.036818Z","shell.execute_reply":"2022-09-19T22:47:26.172164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"---","metadata":{}},{"cell_type":"markdown","source":"# Scaling\n\nFirst, let's use DBScan, a type of unsupervised learning, to classify the riders. We assume that models with a small number of occurrences should be classified as noise, and models with a large number of occurrences should be classified into multiple clusters. This analysis can be done statistically, but clustering is done using unsupervised learning.","metadata":{}},{"cell_type":"code","source":"temp_df = analyze_df.copy()\ndummy_df = pd.get_dummies(temp_df[target_cat_col_lst[1:]]) # drop `race_id`\ntemp_df = temp_df[target_num_col_lst[:-2]].join(dummy_df) # drop `horse_id` and `position_at_finish`\ntemp_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:26.174162Z","iopub.execute_input":"2022-09-19T22:47:26.174990Z","iopub.status.idle":"2022-09-19T22:47:26.253347Z","shell.execute_reply.started":"2022-09-19T22:47:26.174957Z","shell.execute_reply":"2022-09-19T22:47:26.252262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ss = StandardScaler()\nX = temp_df.values\nX = ss.fit_transform(X)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:26.254730Z","iopub.execute_input":"2022-09-19T22:47:26.255287Z","iopub.status.idle":"2022-09-19T22:47:26.303497Z","shell.execute_reply.started":"2022-09-19T22:47:26.255256Z","shell.execute_reply":"2022-09-19T22:47:26.302520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clustering\n## Jockey\nLet's determine the number of clusters and the necessary parameters to be determined by DBScan.","metadata":{}},{"cell_type":"code","source":"for eps in np.arange(15,30,1):\n    for minPts in range(2,3):\n        dbscan = DBSCAN(eps=eps,min_samples=minPts).fit(X)\n        y_dbscan = dbscan.labels_\n        print(f\"eps:{eps}minPts:{minPts} :Converge Condition = blow row number * 1%\")\n        if len(np.where(y_dbscan ==-1)[0])< X.shape[0]*0.01:\n            print(f\"Number of outlier:{len(np.where(y_dbscan ==-1)[0])}\")\n            print(f\"Number of cluster:{np.max(y_dbscan)}\")\n            print(f\"{'----'*30}\")\n        else:\n            print(f\"{'----'*30}\")","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:47:26.305068Z","iopub.execute_input":"2022-09-19T22:47:26.305396Z","iopub.status.idle":"2022-09-19T22:48:43.803976Z","shell.execute_reply.started":"2022-09-19T22:47:26.305368Z","shell.execute_reply":"2022-09-19T22:48:43.802683Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The number of outliers does not decrease even if `eps` exceeds 20, so 20 is used. `min_samples` is set to 2 because the number of outliers increases if min_samples is anything other than 2.","metadata":{}},{"cell_type":"code","source":"dbs = DBSCAN(eps=20, min_samples=2).fit(X)\nanalyze_df['cluster']=dbs.labels_\nanalyze_df['cluster'] = analyze_df['cluster']\nlen(analyze_df['cluster'].unique())","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:48:43.805372Z","iopub.execute_input":"2022-09-19T22:48:43.805798Z","iopub.status.idle":"2022-09-19T22:48:48.778089Z","shell.execute_reply.started":"2022-09-19T22:48:43.805767Z","shell.execute_reply":"2022-09-19T22:48:48.777106Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's make sure that riders with a small number of entries are classified as outliers as targeted.","metadata":{}},{"cell_type":"code","source":"for jockey in analyze_df['jockey'].unique():\n    idx = analyze_df.query('jockey==@jockey').index\n    analyze_df.loc[idx,'is_multi_cluster'] = len(analyze_df.query('jockey==@jockey')['cluster'].unique())\n\ntemp = analyze_df.query(\"is_multi_cluster==1 & cluster==-1\").groupby('jockey')['race_id'].count().to_frame().reset_index()\ntemp = temp.rename(columns={'race_id':'Count'})\npx.bar(temp,x='jockey',y='Count')","metadata":{"_kg_hide-output":false,"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:48:48.779315Z","iopub.execute_input":"2022-09-19T22:48:48.779615Z","iopub.status.idle":"2022-09-19T22:48:50.078707Z","shell.execute_reply.started":"2022-09-19T22:48:48.779588Z","shell.execute_reply":"2022-09-19T22:48:50.077594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Do the jockey in the most clusters actually appear more often?","metadata":{}},{"cell_type":"code","source":"temp = analyze_df.groupby(['track_id','position_at_finish','course_type','is_multi_cluster','jockey'])['race_id'].count().to_frame().reset_index()\ntemp = temp.rename(columns={'race_id':'Count'})\n\ntemp2 = analyze_df.groupby(['track_id','jockey'])['race_id'].count().to_frame().reset_index()\ntemp2 = temp2.rename(columns={'race_id':'TTL Count'})\n\n# merge dataframe\ntemp = temp.merge(temp2,on=['track_id','jockey'])\ntemp['Ratio'] = 100*temp['Count']/temp['TTL Count']\n\npx.bar(temp,x=target_col,y='Count'\n       ,facet_col='track_id',facet_row='course_type'\n       ,color='is_multi_cluster'\n       ,hover_data=['Count','is_multi_cluster']\n      ,height=700,width=900)","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-09-19T22:48:50.080247Z","iopub.execute_input":"2022-09-19T22:48:50.080819Z","iopub.status.idle":"2022-09-19T22:48:50.452138Z","shell.execute_reply.started":"2022-09-19T22:48:50.080788Z","shell.execute_reply":"2022-09-19T22:48:50.450873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"By using DBScan, it appears that we have been able to classify winning jockeys at various racetracks.","metadata":{}},{"cell_type":"markdown","source":"## Horse","metadata":{"execution":{"iopub.status.busy":"2022-09-19T04:04:14.021772Z","iopub.execute_input":"2022-09-19T04:04:14.022228Z","iopub.status.idle":"2022-09-19T04:04:14.059174Z","shell.execute_reply.started":"2022-09-19T04:04:14.022188Z","shell.execute_reply":"2022-09-19T04:04:14.058003Z"}}},{"cell_type":"markdown","source":"### GMM \n\nFirst, let's try clustering with GMM. We can only get fairly rough classification results, as the data is quite detailed, right? I am thinking. Therefore, I thinned out the data every 4 seconds before clustering.","metadata":{}},{"cell_type":"code","source":"temp_df = analyze_df[['race_id','program_number','course_type','odds','purse','distance_id','track_condition','is_multi_cluster','horse_id']].copy()\ntemp_df = temp_df.merge(tracking_df,on=['race_id','program_number'])\n\ntemp_df['trakus_index_mod'] = temp_df['trakus_index'] % 16\ntemp_df = temp_df[temp_df['trakus_index_mod']==0]\n\ntemp_df = temp_df.drop(columns=['race_id','race_date','trakus_index_mod'])\ncat_col =['track_id','course_type','track_condition','program_number']\ndummy_df = pd.get_dummies(temp_df[cat_col]) \ntemp_df = temp_df.drop(columns=cat_col).join(dummy_df)\n\nss = StandardScaler()\nX = temp_df.values\nX = ss.fit_transform(X)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:48:50.453898Z","iopub.execute_input":"2022-09-19T22:48:50.454256Z","iopub.status.idle":"2022-09-19T22:48:55.479581Z","shell.execute_reply.started":"2022-09-19T22:48:50.454226Z","shell.execute_reply":"2022-09-19T22:48:55.478354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clusters = np.arange(2,10)\naic = []\nbic = []\n\nfor i in clusters:\n    gmm = GaussianMixture(n_components=i, random_state=3, verbose=0, n_init=1)\n    gmm.fit(X)\n    aic.append(gmm.aic(X))\n    bic.append(gmm.bic(X))\n    if i % 10 ==0:\n        print(f'{i}times finish')\n\ngm = list(zip(clusters, aic, bic))\ngm_df = pd.DataFrame(gm, columns=['clusters', 'aic_score', 'bic_score'])","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:48:55.481055Z","iopub.execute_input":"2022-09-19T22:48:55.481496Z","iopub.status.idle":"2022-09-19T22:51:30.176202Z","shell.execute_reply.started":"2022-09-19T22:48:55.481464Z","shell.execute_reply":"2022-09-19T22:51:30.174910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.line(gm_df, x='clusters', y='aic_score',title='Akaike Information Criterion Score')","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:51:30.177661Z","iopub.execute_input":"2022-09-19T22:51:30.178008Z","iopub.status.idle":"2022-09-19T22:51:30.253940Z","shell.execute_reply.started":"2022-09-19T22:51:30.177967Z","shell.execute_reply":"2022-09-19T22:51:30.252771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"px.line(gm_df, x='clusters', y='bic_score',title='Baysian Information Criterion Score')","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:51:30.255082Z","iopub.execute_input":"2022-09-19T22:51:30.255392Z","iopub.status.idle":"2022-09-19T22:51:30.312186Z","shell.execute_reply.started":"2022-09-19T22:51:30.255364Z","shell.execute_reply":"2022-09-19T22:51:30.311153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df = analyze_df[['race_id','program_number','course_type','odds','purse','distance_id','track_condition','is_multi_cluster','horse_id']].copy()\ntemp_df = temp_df.merge(tracking_df,on=['race_id','program_number'])\n\ntemp_df = temp_df.drop(columns=['race_id','race_date'])\ncat_col =['track_id','course_type','track_condition','program_number']\ndummy_df = pd.get_dummies(temp_df[cat_col]) \ntemp_df = temp_df.drop(columns=cat_col).join(dummy_df)\n\nss = StandardScaler()\nX = temp_df.values\nX = ss.fit_transform(X)\n\ngmm = GaussianMixture(n_components=7, random_state=3, verbose=0, n_init=1)\ngmm_preds = gmm.fit_predict(X)\ntracking_df['horse_cluster']=gmm_preds","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:54:49.437528Z","iopub.execute_input":"2022-09-19T22:54:49.437912Z","iopub.status.idle":"2022-09-19T22:57:43.980355Z","shell.execute_reply.started":"2022-09-19T22:54:49.437881Z","shell.execute_reply":"2022-09-19T22:57:43.978704Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp_df=analyze_df.merge(tracking_df[['race_id','program_number','horse_cluster']].drop_duplicates()\n                            ,on=['race_id','program_number']).copy()\ntemp1_df = temp_df.groupby(['track_id','course_type','horse_cluster','position_at_finish'])['race_id'].count().to_frame().reset_index()\ntemp1_df = temp1_df.rename(columns={'race_id':'Count'})\n\ntemp2_df = temp_df.groupby(['track_id','course_type','horse_cluster'])['race_id'].count().to_frame().reset_index()\ntemp2_df = temp2_df.rename(columns={'race_id':'TTL Count'})\ntemp2_df = temp2_df.merge(temp1_df, on=['horse_cluster','track_id','course_type'])\n\ntemp2_df['Ratio'] = 100*temp2_df['Count'] / temp2_df['TTL Count']\n\n\npx.bar(temp2_df,x='position_at_finish',y='Count'\n       ,facet_col='track_id',facet_row='course_type'\n       ,color='horse_cluster'\n       ,hover_data=['Count','horse_cluster']\n      ,height=700,width=900)","metadata":{"execution":{"iopub.status.busy":"2022-09-19T22:57:43.982659Z","iopub.execute_input":"2022-09-19T22:57:43.983027Z","iopub.status.idle":"2022-09-19T22:57:46.133724Z","shell.execute_reply.started":"2022-09-19T22:57:43.982977Z","shell.execute_reply":"2022-09-19T22:57:46.132664Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}