{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-18T05:15:52.791776Z","iopub.execute_input":"2022-08-18T05:15:52.792333Z","iopub.status.idle":"2022-08-18T05:15:52.824231Z","shell.execute_reply.started":"2022-08-18T05:15:52.792245Z","shell.execute_reply":"2022-08-18T05:15:52.823124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Working with Pandas","metadata":{}},{"cell_type":"markdown","source":"### Tacking Table ","metadata":{}},{"cell_type":"code","source":"%%time\ntracking_table = pd.read_csv(\"/kaggle/input/big-data-derby-2022/nyra_tracking_table.csv\")\ntracking_table.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:16:05.138981Z","iopub.execute_input":"2022-08-18T05:16:05.139407Z","iopub.status.idle":"2022-08-18T05:16:12.495489Z","shell.execute_reply.started":"2022-08-18T05:16:05.139370Z","shell.execute_reply":"2022-08-18T05:16:12.494328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracking_table.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:22:14.130981Z","iopub.execute_input":"2022-08-18T05:22:14.131382Z","iopub.status.idle":"2022-08-18T05:22:14.779173Z","shell.execute_reply.started":"2022-08-18T05:22:14.131350Z","shell.execute_reply":"2022-08-18T05:22:14.777987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tracking_table.info()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:22:37.052089Z","iopub.execute_input":"2022-08-18T05:22:37.053258Z","iopub.status.idle":"2022-08-18T05:22:37.070473Z","shell.execute_reply.started":"2022-08-18T05:22:37.053213Z","shell.execute_reply":"2022-08-18T05:22:37.069563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntracking_table.nunique()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:26:01.001709Z","iopub.execute_input":"2022-08-18T05:26:01.002131Z","iopub.status.idle":"2022-08-18T05:26:03.841511Z","shell.execute_reply.started":"2022-08-18T05:26:01.002096Z","shell.execute_reply":"2022-08-18T05:26:03.840214Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Let's do some analysis by using just 2 line of code ","metadata":{}},{"cell_type":"code","source":"import pandas_profiling as pf","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:37:20.833088Z","iopub.execute_input":"2022-08-18T05:37:20.833513Z","iopub.status.idle":"2022-08-18T05:37:20.841252Z","shell.execute_reply.started":"2022-08-18T05:37:20.833479Z","shell.execute_reply":"2022-08-18T05:37:20.840266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ntracking_data_analysis = pf.ProfileReport(tracking_table)\ntracking_data_analysis","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:54:33.779712Z","iopub.execute_input":"2022-08-18T05:54:33.780781Z","iopub.status.idle":"2022-08-18T05:56:45.573921Z","shell.execute_reply.started":"2022-08-18T05:54:33.780732Z","shell.execute_reply":"2022-08-18T05:56:45.569328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## start table","metadata":{}},{"cell_type":"code","source":"%%time\nstart_table = pd.read_csv(\"/kaggle/input/big-data-derby-2022/nyra_start_table.csv\")\nstart_table.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:17:00.356729Z","iopub.execute_input":"2022-08-18T05:17:00.357524Z","iopub.status.idle":"2022-08-18T05:17:00.398026Z","shell.execute_reply.started":"2022-08-18T05:17:00.357488Z","shell.execute_reply":"2022-08-18T05:17:00.396858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nstart_table_analysis = pf.ProfileReport(start_table)\nstart_table_analysis","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:43:48.781567Z","iopub.execute_input":"2022-08-18T05:43:48.782020Z","iopub.status.idle":"2022-08-18T05:43:55.552516Z","shell.execute_reply.started":"2022-08-18T05:43:48.781983Z","shell.execute_reply":"2022-08-18T05:43:55.551471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## race table","metadata":{}},{"cell_type":"code","source":"%%time\nrace_table = pd.read_csv(\"/kaggle/input/big-data-derby-2022/nyra_race_table.csv\")\nrace_table.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:18:27.079176Z","iopub.execute_input":"2022-08-18T05:18:27.079666Z","iopub.status.idle":"2022-08-18T05:18:27.105933Z","shell.execute_reply.started":"2022-08-18T05:18:27.079626Z","shell.execute_reply":"2022-08-18T05:18:27.104622Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nrace_table_analysis = pf.ProfileReport(race_table)\nrace_table_analysis","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:44:37.926694Z","iopub.execute_input":"2022-08-18T05:44:37.927141Z","iopub.status.idle":"2022-08-18T05:44:49.794999Z","shell.execute_reply.started":"2022-08-18T05:44:37.927098Z","shell.execute_reply":"2022-08-18T05:44:49.793849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## complete data","metadata":{}},{"cell_type":"code","source":"%%time\ncomplete_table = pd.read_csv(\"/kaggle/input/big-data-derby-2022/nyra_2019_complete.csv\")\ncomplete_table.head()","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:19:02.712278Z","iopub.execute_input":"2022-08-18T05:19:02.712954Z","iopub.status.idle":"2022-08-18T05:19:18.235953Z","shell.execute_reply.started":"2022-08-18T05:19:02.712915Z","shell.execute_reply":"2022-08-18T05:19:18.234858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ncomplete_data_analysis = pf.ProfileReport(complete_table)\ncomplete_data_analysis","metadata":{"execution":{"iopub.status.busy":"2022-08-18T05:45:44.441614Z","iopub.execute_input":"2022-08-18T05:45:44.442064Z","iopub.status.idle":"2022-08-18T05:52:28.480286Z","shell.execute_reply.started":"2022-08-18T05:45:44.442024Z","shell.execute_reply":"2022-08-18T05:52:28.478582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import ipywidgets as widgets\nimport matplotlib.pyplot as plt\nimport plotly.graph_objects as go\nimport os\nimport seaborn as sns\n\n\nfrom datetime import datetime as dt\nfrom IPython.display import display, HTML","metadata":{"execution":{"iopub.status.busy":"2022-08-18T06:14:33.178867Z","iopub.execute_input":"2022-08-18T06:14:33.179371Z","iopub.status.idle":"2022-08-18T06:14:33.194381Z","shell.execute_reply.started":"2022-08-18T06:14:33.179333Z","shell.execute_reply":"2022-08-18T06:14:33.193448Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thanks to -> https://www.kaggle.com/code/alejopaullier/big-data-derby-2022-complete-eda\n\nimport matplotlib as mpl\n\n# source: https://stackoverflow.com/questions/32485907/matplotlib-and-numpy-create-a-calendar-heatmap\n\nDAYS = ['Sun.', 'Mon.', 'Tues.', 'Wed.', 'Thurs.', 'Fri.', 'Sat.']\nMONTHS = ['Jan.', 'Feb.', 'Mar.', 'Apr.', 'May', 'June', 'July', 'Aug.', 'Sept.', 'Oct.', 'Nov.', 'Dec.']\n\n\ndef date_heatmap(series, start=None, end=None, mean=False, ax=None, **kwargs):\n    '''Plot a calendar heatmap given a datetime series.\n\n    Arguments:\n        series (pd.Series):\n            A series of numeric values with a datetime index. Values occurring\n            on the same day are combined by sum.\n        start (Any):\n            The first day to be considered in the plot. The value can be\n            anything accepted by :func:`pandas.to_datetime`. The default is the\n            earliest date in the data.\n        end (Any):\n            The last day to be considered in the plot. The value can be\n            anything accepted by :func:`pandas.to_datetime`. The default is the\n            latest date in the data.\n        mean (bool):\n            Combine values occurring on the same day by mean instead of sum.\n        ax (matplotlib.Axes or None):\n            The axes on which to draw the heatmap. The default is the current\n            axes in the :module:`~matplotlib.pyplot` API.\n        **kwargs:\n            Forwarded to :meth:`~matplotlib.Axes.pcolormesh` for drawing the\n            heatmap.\n\n    Returns:\n        matplotlib.collections.Axes:\n            The axes on which the heatmap was drawn. This is set as the current\n            axes in the `~matplotlib.pyplot` API.\n    '''\n    # Combine values occurring on the same day.\n    dates = series.index.floor('D')\n    group = series.groupby(dates)\n    series = group.mean() if mean else group.sum()\n\n    # Parse start/end, defaulting to the min/max of the index.\n    start = pd.to_datetime(start or series.index.min())\n    end = pd.to_datetime(end or series.index.max())\n\n    # We use [start, end) as a half-open interval below.\n    end += np.timedelta64(1, 'D')\n\n    # Get the previous/following Sunday to start/end.\n    # Pandas and numpy day-of-week conventions are Monday=0 and Sunday=6.\n    start_sun = start - np.timedelta64((start.dayofweek + 1) % 7, 'D')\n    end_sun = end + np.timedelta64(7 - end.dayofweek - 1, 'D')\n\n    # Create the heatmap and track ticks.\n    num_weeks = (end_sun - start_sun).days // 7\n    heatmap = np.zeros((7, num_weeks))\n    ticks = {}  # week number -> month name\n    for week in range(num_weeks):\n        for day in range(7):\n            date = start_sun + np.timedelta64(7 * week + day, 'D')\n            if date.day == 1:\n                ticks[week] = MONTHS[date.month - 1]\n            if date.dayofyear == 1:\n                ticks[week] += f'\\n{date.year}'\n            if start <= date < end:\n                heatmap[day, week] = series.get(date, 0)\n\n    # Get the coordinates, offset by 0.5 to align the ticks.\n    y = np.arange(8) - 0.5\n    x = np.arange(num_weeks + 1) - 0.5\n\n    # Plot the heatmap. Prefer pcolormesh over imshow so that the figure can be\n    # vectorized when saved to a compatible format. We must invert the axis for\n    # pcolormesh, but not for imshow, so that it reads top-bottom, left-right.\n    ax = ax or plt.gca()\n    mesh = ax.pcolormesh(x, y, heatmap, **kwargs)\n    ax.invert_yaxis()\n\n    # Set the ticks.\n    ax.set_xticks(list(ticks.keys()))\n    ax.set_xticklabels(list(ticks.values()))\n    ax.set_yticks(np.arange(7))\n    ax.set_yticklabels(DAYS)\n\n    # Set the current image and axes in the pyplot API.\n    plt.sca(ax)\n    plt.sci(mesh)\n\n    return ax\n\n\n# # Get some data, a series of values with datetime index.\n# data = np.random.randint(5, size=365)\n# data = pd.Series(data)\n# data.index = pd.date_range(start='2017-01-01', end='2017-12-31', freq='1D')\n\nrace_table[\"race_date\"] =  pd.to_datetime(race_table[\"race_date\"], format='%Y-%m-%d')\ndata = race_table[\"race_date\"].value_counts()\n\n# Create the figure. For the aspect ratio, one year is 7 days by 53 weeks.\n# We widen it further to account for the tick labels and color bar.\nfigsize = plt.figaspect(7 / 56)\nfig = plt.figure(figsize=(figsize))\n\n# Plot the heatmap with a color bar.\nax = date_heatmap(data, edgecolor='black')\nplt.colorbar(ticks=range(max(data.values)), pad=0.02)\ncmap = mpl.cm.get_cmap('Blues', 5)\nplt.set_cmap(cmap)\n# Force the cells to be square. If this is set, the size of the color bar\n# may look weird compared to the size of the heatmap. That can be corrected\n# by the aspect ratio of the figure or scale of the color bar.\nax.set_aspect('equal')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-18T06:16:39.506369Z","iopub.execute_input":"2022-08-18T06:16:39.507424Z","iopub.status.idle":"2022-08-18T06:16:39.647652Z","shell.execute_reply.started":"2022-08-18T06:16:39.507370Z","shell.execute_reply":"2022-08-18T06:16:39.646440Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}