{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceType":"competition","sourceId":130062,"databundleVersionId":15606741}],"dockerImageVersionId":31286,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:49.170842Z","iopub.execute_input":"2026-02-24T07:59:49.171102Z","iopub.status.idle":"2026-02-24T07:59:49.467279Z","shell.execute_reply.started":"2026-02-24T07:59:49.171081Z","shell.execute_reply":"2026-02-24T07:59:49.466173Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"raw_train = pd.read_csv('/kaggle/input/bidding-predictions-for-construction/raw_train.csv')\nraw_test = pd.read_csv('/kaggle/input/bidding-predictions-for-construction/raw_test.csv')\ndf_train = pd.read_csv('/kaggle/input/bidding-predictions-for-construction/train_summary.csv')\ndf_test = pd.read_csv('/kaggle/input/bidding-predictions-for-construction/test.csv')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:49.468877Z","iopub.execute_input":"2026-02-24T07:59:49.469249Z","iopub.status.idle":"2026-02-24T07:59:52.789625Z","shell.execute_reply.started":"2026-02-24T07:59:49.469199Z","shell.execute_reply":"2026-02-24T07:59:52.788627Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# TOTAL BID (prediction target)","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 5))\nplt.hist(df_train['total_bid'], bins=30)\nplt.title('Histogram of total_bid')\nplt.xlabel('total_bid')\nplt.ylabel('Frequency')\nplt.grid(True)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:52.790584Z","iopub.execute_input":"2026-02-24T07:59:52.791625Z","iopub.status.idle":"2026-02-24T07:59:53.036839Z","shell.execute_reply.started":"2026-02-24T07:59:52.791513Z","shell.execute_reply":"2026-02-24T07:59:53.035679Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Total bid is skewed, let's look at a log-scaled version:","metadata":{}},{"cell_type":"code","source":"plt.figure(figsize=(8, 5))\nplt.hist(np.log1p(df_train['total_bid']), bins=30)\nplt.title('Histogram of log1p(total_bid)')\nplt.xlabel('log1p(total_bid)')\nplt.ylabel('Frequency')\nplt.grid(True)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:53.038105Z","iopub.execute_input":"2026-02-24T07:59:53.038404Z","iopub.status.idle":"2026-02-24T07:59:53.197699Z","shell.execute_reply.started":"2026-02-24T07:59:53.038375Z","shell.execute_reply":"2026-02-24T07:59:53.196673Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Let's see when the bids in train happened\n","metadata":{}},{"cell_type":"code","source":"df_train['bid_date'] = pd.to_datetime(df_train['bid_date'])\n\nbids_per_month = df_train.set_index('bid_date').resample('ME').size()\n\nplt.figure(figsize=(12, 5))\nplt.plot(bids_per_month.index, bids_per_month.values)\n\nplt.title('Number of Bids Per Month')\nplt.xlabel('Date')\nplt.ylabel('Number of Bids')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:53.199987Z","iopub.execute_input":"2026-02-24T07:59:53.200276Z","iopub.status.idle":"2026-02-24T07:59:53.454899Z","shell.execute_reply.started":"2026-02-24T07:59:53.200250Z","shell.execute_reply":"2026-02-24T07:59:53.454020Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Bid number dips in summer. Lets look by month:","metadata":{}},{"cell_type":"code","source":"import calendar\ndf_train['bid_month_name'] = df_train['bid_date'].dt.month_name()\n\n# Ensure correct calendar order\nmonth_order = list(calendar.month_name)[1:]\n\nbids_per_month = (\n    df_train['bid_month_name']\n    .value_counts()\n    .reindex(month_order)\n)\n\nplt.figure(figsize=(12, 5))\nplt.bar(bids_per_month.index, bids_per_month.values)\n\nplt.xticks(rotation=45)\nplt.xlabel('Month')\nplt.ylabel('Number of Bids')\nplt.title('Number of Bids per Month (All Years Combined)')\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:53.455969Z","iopub.execute_input":"2026-02-24T07:59:53.456373Z","iopub.status.idle":"2026-02-24T07:59:53.626522Z","shell.execute_reply.started":"2026-02-24T07:59:53.456334Z","shell.execute_reply":"2026-02-24T07:59:53.625633Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Let's see what happens in TEST","metadata":{}},{"cell_type":"code","source":"# Ensure datetime\ndf_test['bid_date'] = pd.to_datetime(df_test['bid_date'])\n\nbids_per_month_time = (df_test.set_index('bid_date').resample('ME').size())\n\ndf_test['month_name'] = df_test['bid_date'].dt.month_name()\n\nmonth_order = list(calendar.month_name)[1:]\n\nbids_per_month_season = (df_test['month_name'].value_counts().reindex(month_order))\n\nfig, axes = plt.subplots(1, 2, figsize=(16, 5))\n\n# Timeline plot\naxes[0].plot(bids_per_month_time.index, bids_per_month_time.values)\naxes[0].set_title('Bids Per Month (Timeline)')\naxes[0].set_xlabel('Date')\naxes[0].set_ylabel('Number of Bids')\n\n# Seasonality bar plot\naxes[1].bar(bids_per_month_season.index, bids_per_month_season.values)\naxes[1].set_title('Bids Per Month (Ignoring Year)')\naxes[1].set_xlabel('Month')\naxes[1].set_ylabel('Number of Bids')\naxes[1].tick_params(axis='x', rotation=45)\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:53.627549Z","iopub.execute_input":"2026-02-24T07:59:53.627805Z","iopub.status.idle":"2026-02-24T07:59:53.974715Z","shell.execute_reply.started":"2026-02-24T07:59:53.627777Z","shell.execute_reply":"2026-02-24T07:59:53.973752Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Ok, so test is 2025, while train is 2018-2024. Test distribution across months is similar to train, with no bids at all in June\n\n### Let's look at bid amounts per year","metadata":{}},{"cell_type":"code","source":"df_train['bid_year'] = df_train['bid_date'].dt.year\n\nyears = sorted(df_train['bid_year'].unique())\ndata = [np.log1p(df_train[df_train['bid_year'] == y]['total_bid']) for y in years]\n\n# draw boxplot\nplt.figure(figsize=(12, 6))\nplt.boxplot(data, tick_labels=years, showfliers=False)\n\nplt.xlabel('Year')\nplt.ylabel('Total Bid')\nplt.title('Distribution of Log(Total Bid) per Year')\nplt.xticks(rotation=45)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:53.975896Z","iopub.execute_input":"2026-02-24T07:59:53.976146Z","iopub.status.idle":"2026-02-24T07:59:54.133461Z","shell.execute_reply.started":"2026-02-24T07:59:53.976122Z","shell.execute_reply":"2026-02-24T07:59:54.132590Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### No significant change. Either no effect of inflation, or projects became smaller?\n\n### Lets look per month","metadata":{}},{"cell_type":"code","source":"month_names = list(calendar.month_name)\ndata = [np.log1p(df_train[df_train['bid_month_name'] == m]['total_bid']) for m in month_names]\n\n# draw boxplot\nplt.figure(figsize=(12, 6))\nplt.boxplot(data, tick_labels=month_names, showfliers=False)\n\nplt.xlabel('Month')\nplt.ylabel('Total Bid')\nplt.title('Distribution of Log(Total Bid) per Month')\nplt.xticks(rotation=45)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:54.134365Z","iopub.execute_input":"2026-02-24T07:59:54.134573Z","iopub.status.idle":"2026-02-24T07:59:54.335778Z","shell.execute_reply.started":"2026-02-24T07:59:54.134552Z","shell.execute_reply":"2026-02-24T07:59:54.334935Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now let's examine contractors","metadata":{}},{"cell_type":"code","source":"train_contractors = set(df_train['contractor_id'])\ntest_contractors = set(df_test['contractor_id'])\nprint (f\"TRAIN: Number of unique contractors: {len(train_contractors)}\")\nprint (f\"TEST: Number of unique contractors: {len(test_contractors)}\")\nprint (f\"Number of train contractors not appearing in test: {len(train_contractors.difference(test_contractors))}\")\nnew_test_contractors = test_contractors.difference(train_contractors)\nprint (f\"Number of test contractors not appearing in train: {new_test_contractors}\")\nprint(f\"Number of contracts of new contractors in test: {df_test[df_test['contractor_id'].isin(new_test_contractors)].shape[0]}\")\nprint(f\"Ratio of contracts of new contractors in test: {df_test['contractor_id'].isin(new_test_contractors).mean()}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T08:02:30.329035Z","iopub.execute_input":"2026-02-24T08:02:30.329760Z","iopub.status.idle":"2026-02-24T08:02:30.337309Z","shell.execute_reply.started":"2026-02-24T08:02:30.329731Z","shell.execute_reply":"2026-02-24T08:02:30.336467Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### I would want to see if new contractors appear in other years\nBecause I will do rolling validation (train on 2018-2021, validate on 2022. Train on 2018-2022, validate on 2023. Train on 2018-2023, validate on 2024)","metadata":{}},{"cell_type":"code","source":"for val_year in [2022, 2023, 2024]:\n    train_contractors = set(df_train[df_train['bid_date'].dt.year < val_year]['contractor_id'])\n    val_contractors = set(df_train[df_train['bid_date'].dt.year == val_year]['contractor_id'])\n    print (f\"Number of train contractors (2018-{val_year}): {len(train_contractors)}\")\n    print (f\"Number of val contractors not appearing in train for {val_year}: {len(val_contractors.difference(train_contractors))}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:54.352015Z","iopub.execute_input":"2026-02-24T07:59:54.352274Z","iopub.status.idle":"2026-02-24T07:59:54.381784Z","shell.execute_reply.started":"2026-02-24T07:59:54.352251Z","shell.execute_reply":"2026-02-24T07:59:54.380875Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# count number of projects per contractor in train\ntrain_counts = df_train.groupby('contractor_id').size()\n\n# count number of projects per contractor in test\ntest_counts = df_test.groupby('contractor_id').size()\n\n# create side-by-side plots\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# histogram for train\naxes[0].hist(train_counts, bins=30)\naxes[0].set_title('Train: Projects per Contractor')\naxes[0].set_xlabel('Number of Projects')\naxes[0].set_ylabel('Number of Contractors')\n\n# histogram for test\naxes[1].hist(test_counts, bins=30)\naxes[1].set_title('Test: Projects per Contractor')\naxes[1].set_xlabel('Number of Projects')\naxes[1].set_ylabel('Number of Contractors')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:54.382693Z","iopub.execute_input":"2026-02-24T07:59:54.382910Z","iopub.status.idle":"2026-02-24T07:59:54.717548Z","shell.execute_reply.started":"2026-02-24T07:59:54.382890Z","shell.execute_reply":"2026-02-24T07:59:54.716628Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Lots of contractors with very few bids. Lets focus on low-bid contractors:","metadata":{}},{"cell_type":"code","source":"# keep only contractors with <=10 projects\ntrain_small = train_counts[train_counts <= 10]\ntest_small = test_counts[test_counts <= 10]\n\n# count how many contractors have 1,2,...,10 projects\ntrain_freq = train_small.value_counts().reindex(range(1, 11), fill_value=0).sort_index()\ntest_freq = test_small.value_counts().reindex(range(1, 11), fill_value=0).sort_index()\n\n# side-by-side bar chart\nx = range(1, 11)\nwidth = 0.4\n\nplt.figure(figsize=(12, 5))\nplt.bar([i - width/2 for i in x], train_freq.values, width=width, label='Train')\nplt.bar([i + width/2 for i in x], test_freq.values, width=width, label='Test')\n\nplt.xticks(x)\nplt.xlabel('Number of Bids per Contractor')\nplt.ylabel('Number of Contractors')\nplt.title('Contractors with 1–10 Bids')\nplt.legend()\n\nplt.show()","metadata":{"trusted":true,"_kg_hide-input":false,"execution":{"iopub.status.busy":"2026-02-24T07:59:54.719116Z","iopub.execute_input":"2026-02-24T07:59:54.719567Z","iopub.status.idle":"2026-02-24T07:59:54.888093Z","shell.execute_reply.started":"2026-02-24T07:59:54.719522Z","shell.execute_reply":"2026-02-24T07:59:54.887177Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Now job_category_description ","metadata":{},"attachments":{}},{"cell_type":"code","source":"train_categories = set(df_train['job_category_description'])\ntest_categories = set(df_test['job_category_description'])\nprint (f\"TRAIN: Number of unique categories: {len(train_categories)}\")\nprint (f\"TEST: Number of unique categories: {len(test_categories)}\")\nprint (f\"Number of train categories not appearing in test: {len(train_categories.difference(test_categories))}\")\nprint (f\"Number of test categories not appearing in train: {len(test_categories.difference(train_categories))}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:54.890758Z","iopub.execute_input":"2026-02-24T07:59:54.891018Z","iopub.status.idle":"2026-02-24T07:59:54.897741Z","shell.execute_reply.started":"2026-02-24T07:59:54.890996Z","shell.execute_reply":"2026-02-24T07:59:54.896292Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# count categories\ntrain_counts = df_train['job_category_description'].value_counts()\ntest_counts = df_test['job_category_description'].value_counts()\n\n# align categories so both have same index\nall_categories = sorted(set(train_counts.index).union(set(test_counts.index)))\ntrain_counts = train_counts.reindex(all_categories, fill_value=0)\ntest_counts = test_counts.reindex(all_categories, fill_value=0)\n\n# bar positions\nx = range(len(all_categories))\nwidth = 0.4\n\nplt.figure(figsize=(14, 6))\nplt.bar([i - width/2 for i in x], train_counts.values, width=width, label='Train')\nplt.bar([i + width/2 for i in x], test_counts.values, width=width, label='Test')\n\nplt.xticks(x, all_categories, rotation=45, ha='right')\nplt.xlabel('Job Category')\nplt.ylabel('Number of Bids')\nplt.title('Job Category Distribution: Train vs Test')\nplt.legend()\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:54.899524Z","iopub.execute_input":"2026-02-24T07:59:54.900408Z","iopub.status.idle":"2026-02-24T07:59:55.208104Z","shell.execute_reply.started":"2026-02-24T07:59:54.900373Z","shell.execute_reply":"2026-02-24T07:59:55.207234Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Perhaps some of those categories can be grouped together. Need to look into it later.\n\n### Let's see if there is a difference in bids per category","metadata":{}},{"cell_type":"code","source":"data = [np.log1p(df_train[df_train['job_category_description'] == c]['total_bid']) for c in all_categories]\n\n# draw boxplot\nplt.figure(figsize=(18, 9))\nplt.boxplot(data, tick_labels=all_categories, showfliers=False)\n\nplt.xlabel('Month')\nplt.ylabel('Total Bid')\nplt.title('Distribution of Log(Total Bid) per Month')\nplt.xticks(rotation=45)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:55.208952Z","iopub.execute_input":"2026-02-24T07:59:55.209165Z","iopub.status.idle":"2026-02-24T07:59:55.571350Z","shell.execute_reply.started":"2026-02-24T07:59:55.209145Z","shell.execute_reply":"2026-02-24T07:59:55.570605Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# Primary Location","metadata":{}},{"cell_type":"code","source":"train_locations = set(df_train['primary_location'])\ntest_locations = set(df_test['primary_location'])\nprint (f\"TRAIN: Number of unique locations: {len(train_locations)}\")\nprint (f\"TEST: Number of unique locations: {len(test_locations)}\")\nprint (f\"Number of train locations not appearing in test: {len(train_locations.difference(test_locations))}\")\nprint (f\"Number of test locations not appearing in train: {len(test_locations.difference(train_locations))}\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:55.573135Z","iopub.execute_input":"2026-02-24T07:59:55.573394Z","iopub.status.idle":"2026-02-24T07:59:55.579776Z","shell.execute_reply.started":"2026-02-24T07:59:55.573372Z","shell.execute_reply":"2026-02-24T07:59:55.578907Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# count number of projects per location in train\ntrain_counts = df_train.groupby('primary_location').size()\n\n# count number of projects per location in test\ntest_counts = df_test.groupby('primary_location').size()\n\n# create side-by-side plots\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# histogram for train\naxes[0].hist(train_counts, bins=30)\naxes[0].set_title('Train: Projects per location')\naxes[0].set_xlabel('Number of Projects')\naxes[0].set_ylabel('Number of location')\n\n# histogram for test\naxes[1].hist(test_counts, bins=30)\naxes[1].set_title('Test: Projects per location')\naxes[1].set_xlabel('Number of Projects')\naxes[1].set_ylabel('Number of location')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:55.580807Z","iopub.execute_input":"2026-02-24T07:59:55.581142Z","iopub.status.idle":"2026-02-24T07:59:55.887083Z","shell.execute_reply.started":"2026-02-24T07:59:55.581106Z","shell.execute_reply":"2026-02-24T07:59:55.886276Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"### Like with contractor, some locations appear only once or very few times","metadata":{}},{"cell_type":"markdown","source":"# Now the raw data\nWhen bidding the contractors broke the bid to components.\nLet's make sure that sum of quantity * amound gives the total bid:","metadata":{}},{"cell_type":"code","source":"raw_train['item_cost'] = raw_train['quantity'] * raw_train['amount']\ngroup_by_row_id = raw_train.groupby('row_id').agg({'item_cost':'sum','total_bid':'first'})\n(group_by_row_id['item_cost'] - group_by_row_id['total_bid']).abs().lt(1).all()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:55.888139Z","iopub.execute_input":"2026-02-24T07:59:55.888483Z","iopub.status.idle":"2026-02-24T07:59:55.994471Z","shell.execute_reply.started":"2026-02-24T07:59:55.888445Z","shell.execute_reply":"2026-02-24T07:59:55.993713Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# PayItems per bid","metadata":{}},{"cell_type":"code","source":"# count number of items per bid in train\ntrain_counts = raw_train.groupby('row_id').size()\n\n# count number of items per bid in test\ntest_counts = raw_test.groupby('row_id').size()\n\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# histogram for train\naxes[0].hist(train_counts, bins=30)\naxes[0].set_title('Train: Items per bid')\naxes[0].set_xlabel('Number of bids')\naxes[0].set_ylabel('Number of items')\n\n# histogram for test\naxes[1].hist(test_counts, bins=30)\naxes[1].set_title('Test: Items per bid')\naxes[1].set_xlabel('Number of bids')\naxes[1].set_ylabel('Number of items')\n\nplt.tight_layout()\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:55.995608Z","iopub.execute_input":"2026-02-24T07:59:55.996022Z","iopub.status.idle":"2026-02-24T07:59:56.477144Z","shell.execute_reply.started":"2026-02-24T07:59:55.995993Z","shell.execute_reply":"2026-02-24T07:59:56.476201Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# category_id\nEvery item in a bid has a category_id, with a description. Let's examine them","metadata":{},"attachments":{}},{"cell_type":"code","source":"print(f\"Distinct category_id: {len(set(raw_train['category_id']))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:56.478172Z","iopub.execute_input":"2026-02-24T07:59:56.478452Z","iopub.status.idle":"2026-02-24T07:59:56.533921Z","shell.execute_reply.started":"2026-02-24T07:59:56.478429Z","shell.execute_reply":"2026-02-24T07:59:56.532822Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# count and sort categories by frequency\ncounts = raw_train['category_description'].value_counts()\nsorted_categories = counts.index\n\n# prepare data and labels\ndata = [raw_train.loc[raw_train['category_description'] == c, 'amount'] for c in sorted_categories]\nlabels = [f\"{c}\\n(n={counts[c]})\" for c in sorted_categories]\n\n# draw boxplot\nplt.figure(figsize=(18, 9))\nplt.boxplot(data, tick_labels=labels, showfliers=False)\n\nplt.xlabel('Category')\nplt.ylabel('Amount')\nplt.title('Distribution of Amount per Category (sorted by frequency)')\nplt.xticks(rotation=90)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:56.535187Z","iopub.execute_input":"2026-02-24T07:59:56.535454Z","iopub.status.idle":"2026-02-24T07:59:58.572804Z","shell.execute_reply.started":"2026-02-24T07:59:56.535432Z","shell.execute_reply":"2026-02-24T07:59:58.572115Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# pay_item_description","metadata":{},"attachments":{}},{"cell_type":"code","source":"print(f\"Distinct pay_item_description: {len(set(raw_train['pay_item_description']))}\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:58.573705Z","iopub.execute_input":"2026-02-24T07:59:58.573945Z","iopub.status.idle":"2026-02-24T07:59:58.639967Z","shell.execute_reply.started":"2026-02-24T07:59:58.573922Z","shell.execute_reply":"2026-02-24T07:59:58.639016Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# count number of projects per contractor in train\ntrain_counts = raw_train.groupby('pay_item_description').size()\n\n# count number of projects per contractor in test\ntest_counts = raw_test.groupby('pay_item_description').size()\n\n# create side-by-side plots\nfig, axes = plt.subplots(1, 2, figsize=(14, 5))\n\n# histogram for train\naxes[0].hist(train_counts, bins=30)\naxes[0].set_title('Train: items per pay_item_description')\naxes[0].set_xlabel('Number of items')\naxes[0].set_ylabel('Number of pay_item_description')\n\n# histogram for test\naxes[1].hist(test_counts, bins=30)\naxes[1].set_title('Test: items per pay_item_description')\naxes[1].set_xlabel('Number of items')\naxes[1].set_ylabel('Number of pay_item_description')\n\nplt.tight_layout()\nplt.show()\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:58.641356Z","iopub.execute_input":"2026-02-24T07:59:58.641751Z","iopub.status.idle":"2026-02-24T07:59:59.045584Z","shell.execute_reply.started":"2026-02-24T07:59:58.641712Z","shell.execute_reply":"2026-02-24T07:59:59.044789Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"# unit_english_id","metadata":{}},{"cell_type":"code","source":"# count and sort categories by frequency\ncounts = raw_train['unit_english_id'].value_counts()\nsorted_uei = counts.index\n\n# prepare data and labels\ndata = [raw_train.loc[raw_train['unit_english_id'] == c, 'amount'] for c in sorted_uei]\nlabels = [f\"{c}\\n(n={counts[c]})\" for c in sorted_uei]\n\n# draw boxplot\nplt.figure(figsize=(18, 9))\nplt.boxplot(data, tick_labels=labels, showfliers=False)\n\nplt.xlabel('unit_english_id')\nplt.ylabel('Amount')\nplt.title('Distribution of Amount per unit_english_id (sorted by frequency)')\nplt.xticks(rotation=90)\n\nplt.show()","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-24T07:59:59.046699Z","iopub.execute_input":"2026-02-24T07:59:59.047013Z","iopub.status.idle":"2026-02-24T08:00:00.835115Z","shell.execute_reply.started":"2026-02-24T07:59:59.046990Z","shell.execute_reply":"2026-02-24T08:00:00.834091Z"}},"outputs":[],"execution_count":null}]}