{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":130062,"databundleVersionId":15606741,"isSourceIdPinned":false,"sourceType":"competition"},{"sourceId":14788057,"sourceType":"datasetVersion","datasetId":9453994}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Loading and previewing the csvs....","metadata":{}},{"cell_type":"code","source":"!pip install -q tabicl","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:03.449957Z","iopub.execute_input":"2026-02-12T16:08:03.450192Z","iopub.status.idle":"2026-02-12T16:08:09.095300Z","shell.execute_reply.started":"2026-02-12T16:08:03.450171Z","shell.execute_reply":"2026-02-12T16:08:09.094565Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np\n\ntest_raw = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/raw_test.csv\")\ntrain_raw = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/raw_train.csv\")\ntest = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/test.csv\")\n\ntrain_summary = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/train_summary.csv\")\n\n# run these at ur discretion to preview the datasets! :)\n# test_raw.head()\n# train_raw.head()\n# train_summary.head()\n# test.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:09.097410Z","iopub.execute_input":"2026-02-12T16:08:09.097753Z","iopub.status.idle":"2026-02-12T16:08:12.946553Z","shell.execute_reply.started":"2026-02-12T16:08:09.097721Z","shell.execute_reply":"2026-02-12T16:08:12.945959Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def summarize_df(df, name, top_n=20):\n    print(f\"\\n=== {name} ===\")\n    print(f\"shape: {df.shape[0]} rows x {df.shape[1]} cols\\n\")\n    \n    # dtypes?\n    dtypes = df.dtypes.astype(str)\n    \n    # nans?\n    miss_count = df.isna().sum()\n    miss_pct = (miss_count / len(df)) * 100\n    \n    # unique counts\n    nunique = df.nunique(dropna=True)\n    \n    # build summary table\n    summary = pd.DataFrame({\n        \"dtype\": dtypes,\n        \"missing_count\": miss_count,\n        \"missing_pct\": miss_pct.round(3),\n        \"n_unique\": nunique\n    }).sort_values([\"missing_count\", \"n_unique\"], ascending=False)\n    \n    display(summary)\n    \n    # numeric stats\n    num_cols = df.select_dtypes(include=[np.number]).columns\n    if len(num_cols) > 0:\n        print(\"\\nNumeric describe (log-safe preview):\")\n        display(df[num_cols].describe().T)\n    \n    # categorical cardinality\n    cat_cols = df.select_dtypes(include=[\"object\", \"category\"]).columns\n    if len(cat_cols) > 0:\n        print(f\"\\nCategorical columns (top {top_n} by cardinality):\")\n        cat_card = nunique[cat_cols].sort_values(ascending=False)\n        display(cat_card.head(top_n))\n    \n    # duplicates\n    dup_count = df.duplicated().sum()\n    print(f\"\\nDuplicate rows: {dup_count}\")\n    \n    return summary\n\nsummary_train_summary = summarize_df(train_summary, \"train_summary\")\nsummary_train_raw = summarize_df(train_raw, \"train_raw\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:12.947464Z","iopub.execute_input":"2026-02-12T16:08:12.947721Z","iopub.status.idle":"2026-02-12T16:08:14.706209Z","shell.execute_reply.started":"2026-02-12T16:08:12.947687Z","shell.execute_reply":"2026-02-12T16:08:14.705596Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"dataset looks nicely organized. moving on...","metadata":{}},{"cell_type":"code","source":"train_summary[\"row_id\"] = train_summary[\"job_id\"] + \"__\" + train_summary[\"contractor_id\"]\ntest[\"row_id\"] = test[\"job_id\"] + \"__\" + test[\"contractor_id\"]\n\n\nfor df in [train_summary, test]:\n    df[\"bid_date\"] = pd.to_datetime(df[\"bid_date\"])\n    df[\"bid_year\"] = df[\"bid_date\"].dt.year\n    df[\"bid_month\"] = df[\"bid_date\"].dt.month\n    df[\"bid_day\"] = df[\"bid_date\"].dt.day\n    df[\"bid_dow\"] = df[\"bid_date\"].dt.dayofweek\n    df[\"bid_week\"] = df[\"bid_date\"].dt.isocalendar().week.astype(int)\n    df[\"bid_date_ordinal\"] = df[\"bid_date\"].map(pd.Timestamp.toordinal)\n\n\ndef build_raw_agg(raw_df):\n    agg_num = raw_df.groupby(\"row_id\").agg(\n        quantity_sum=(\"quantity\", \"sum\"),\n        quantity_mean=(\"quantity\", \"mean\"),\n        quantity_std=(\"quantity\", \"std\"),\n        quantity_min=(\"quantity\", \"min\"),\n        quantity_max=(\"quantity\", \"max\"),\n        num_pay_items_mean=(\"num_pay_items\", \"mean\"),\n        num_pay_items_max=(\"num_pay_items\", \"max\"),\n        num_pay_items_min=(\"num_pay_items\", \"min\"),\n    )\n\n\n    agg_cat = raw_df.groupby(\"row_id\").agg(\n        n_pay_item_id=(\"pay_item_id\", \"nunique\"),\n        n_pay_item_desc=(\"pay_item_description\", \"nunique\"),\n        n_unit_english=(\"unit_english_id\", \"nunique\"),\n        n_category_id=(\"category_id\", \"nunique\"),\n        n_category_desc=(\"category_description\", \"nunique\"),\n        n_job_category=(\"job_category_description\", \"nunique\"),\n    )\n\n\n    agg_cnt = raw_df.groupby(\"row_id\").size().to_frame(\"n_line_items\")\n\n\n    agg = pd.concat([agg_num, agg_cat, agg_cnt], axis=1)\n    return agg\n\ntrain_raw_agg = build_raw_agg(train_raw)\ntest_raw_agg = build_raw_agg(test_raw)\n\ntrain_merged = train_summary.merge(train_raw_agg, on=\"row_id\", how=\"left\")\ntest_merged = test.merge(test_raw_agg, on=\"row_id\", how=\"left\")\n\ntrain_merged.to_csv(\"my_train_dataset.csv\", index=False)\ntest_merged.to_csv(\"my_test_dataset.csv\", index=False)\n\nprint(\"saved: my_train_dataset.csv and my_test_dataset.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:14.707039Z","iopub.execute_input":"2026-02-12T16:08:14.707267Z","iopub.status.idle":"2026-02-12T16:08:15.621637Z","shell.execute_reply.started":"2026-02-12T16:08:14.707246Z","shell.execute_reply":"2026-02-12T16:08:15.620954Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_merged.head(15)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:15.622555Z","iopub.execute_input":"2026-02-12T16:08:15.622902Z","iopub.status.idle":"2026-02-12T16:08:15.639925Z","shell.execute_reply.started":"2026-02-12T16:08:15.622874Z","shell.execute_reply":"2026-02-12T16:08:15.639174Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Scaling:\nnumeric features: z score\ntemporal features: transformed into \n    - relative day (day / number of days in entire time period that the dataset encompasses)\n    - relative day in year (day / total number of days in that year) \n    - relative day in month (day / total number of days in that month)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ninflation = pd.read_csv(\"/kaggle/input/monthly-us-consumer-price-index/CPIAUCSL.csv\")\ninflation[\"observation_date\"] = pd.to_datetime(inflation[\"observation_date\"])\ninflation[\"year\"] = inflation[\"observation_date\"].dt.year\ninflation[\"month\"] = inflation[\"observation_date\"].dt.month\ninflation = inflation.rename(columns={\"CPIAUCSL\": \"inflation_index\"})\n\ntrain_scaled = train_merged.copy()\ntest_scaled = test_merged.copy()\n\ntrain_scaled = train_scaled.merge(\n    inflation[[\"year\",\"month\",\"inflation_index\"]],\n    left_on=[\"bid_year\",\"bid_month\"],\n    right_on=[\"year\",\"month\"],\n    how=\"left\"\n).drop(columns=[\"year\",\"month\"])\n\ntest_scaled = test_scaled.merge(\n    inflation[[\"year\",\"month\",\"inflation_index\"]],\n    left_on=[\"bid_year\",\"bid_month\"],\n    right_on=[\"year\",\"month\"],\n    how=\"left\"\n).drop(columns=[\"year\",\"month\"])\n\ntrain_scaled[\"inflation_index\"] = train_scaled[\"inflation_index\"].ffill().bfill()\ntest_scaled[\"inflation_index\"] = test_scaled[\"inflation_index\"].ffill().bfill()\n\ntrain_scaled[\"bid_date\"] = pd.to_datetime(train_scaled[\"bid_date\"])\ntest_scaled[\"bid_date\"] = pd.to_datetime(test_scaled[\"bid_date\"])\n\nmin_date = train_scaled[\"bid_date\"].min()\nmax_date = train_scaled[\"bid_date\"].max()\ndate_range_days = max((max_date - min_date).days, 1)\n\nfor df in [train_scaled, test_scaled]:\n    df[\"day_relative_to_full_time_period\"] = (\n        (df[\"bid_date\"] - min_date).dt.days / date_range_days\n    ).clip(0, 1)\n\n    day_of_year = df[\"bid_date\"].dt.dayofyear\n    days_in_year = df[\"bid_date\"].dt.is_leap_year.astype(int).map({1: 366, 0: 365})\n    df[\"day_relative_to_full_year\"] = day_of_year / days_in_year\n\n    df[\"day_relative_to_month\"] = df[\"bid_date\"].dt.day / df[\"bid_date\"].dt.days_in_month\n    df[\"day_relative_to_week\"] = df[\"bid_date\"].dt.dayofweek / 6\n\ndate_cols_to_drop = [\n    \"bid_date\", \"bid_year\", \"bid_month\", \"bid_day\", \"bid_dow\", \"bid_week\", \"bid_date_ordinal\"\n]\nfor col in date_cols_to_drop:\n    if col in train_scaled.columns:\n        train_scaled.drop(columns=col, inplace=True)\n    if col in test_scaled.columns:\n        test_scaled.drop(columns=col, inplace=True)\n\ndate_ratio_cols = [\n    \"day_relative_to_full_time_period\",\n    \"day_relative_to_full_year\",\n    \"day_relative_to_month\",\n    \"day_relative_to_week\"\n]\n\ntarget_col = \"total_bid\"\nexclude_cols = set(date_ratio_cols + [target_col])\n\nnum_cols = train_scaled.select_dtypes(include=[np.number]).columns\nnum_cols = [c for c in num_cols if c not in exclude_cols]\n\nfor col in num_cols:\n    train_scaled[col] = np.log1p(train_scaled[col].clip(lower=0))\n    test_scaled[col] = np.log1p(test_scaled[col].clip(lower=0))\n\ntrain_scaled.to_csv(\"train_scaled.csv\", index=False)\ntest_scaled.to_csv(\"test_scaled.csv\", index=False)\n\nprint(\"Saved: train_scaled.csv and test_scaled.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:15.640796Z","iopub.execute_input":"2026-02-12T16:08:15.641150Z","iopub.status.idle":"2026-02-12T16:08:16.086008Z","shell.execute_reply.started":"2026-02-12T16:08:15.641125Z","shell.execute_reply":"2026-02-12T16:08:16.085399Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_scaled.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:16.087922Z","iopub.execute_input":"2026-02-12T16:08:16.088445Z","iopub.status.idle":"2026-02-12T16:08:16.110265Z","shell.execute_reply.started":"2026-02-12T16:08:16.088421Z","shell.execute_reply":"2026-02-12T16:08:16.109653Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_scaled = pd.read_csv(\"train_scaled.csv\")\ntest_scaled = pd.read_csv(\"test_scaled.csv\")\n\nhigh_card_cols = [\"job_id\", \"contractor_id\", \"primary_location\"]\nfor col in high_card_cols:\n    if col in train_scaled.columns:\n        all_vals = pd.concat([train_scaled[col], test_scaled[col]], axis=0).astype(str)\n        codes, uniques = pd.factorize(all_vals, sort=True)\n        train_scaled[col] = codes[:len(train_scaled)]\n        test_scaled[col] = codes[len(train_scaled):]\n\nlow_card_cols = [\"job_category_description\"]\ntrain_encoded = pd.get_dummies(train_scaled, columns=low_card_cols, drop_first=False)\ntest_encoded = pd.get_dummies(test_scaled, columns=low_card_cols, drop_first=False)\n\ntrain_encoded, test_encoded = train_encoded.align(\n    test_encoded, join=\"left\", axis=1, fill_value=0\n)\n\ntarget_col = \"total_bid\"\nif target_col not in train_encoded.columns and target_col in train_scaled.columns:\n    train_encoded[target_col] = train_scaled[target_col]\n\nbool_cols = train_encoded.select_dtypes(include=[\"bool\"]).columns\ntrain_encoded[bool_cols] = train_encoded[bool_cols].astype(int)\n\nbool_cols_test = test_encoded.select_dtypes(include=[\"bool\"]).columns\ntest_encoded[bool_cols_test] = test_encoded[bool_cols_test].astype(int)\n\ntrain_encoded.to_csv(\"train_encoded.csv\", index=False)\ntest_encoded.to_csv(\"test_encoded.csv\", index=False)\n\nprint(\"Saved: train_encoded.csv and test_encoded.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:16.111111Z","iopub.execute_input":"2026-02-12T16:08:16.111338Z","iopub.status.idle":"2026-02-12T16:08:16.582662Z","shell.execute_reply.started":"2026-02-12T16:08:16.111315Z","shell.execute_reply":"2026-02-12T16:08:16.582031Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"id_cols = [\"row_id\", \"job_id\", \"contractor_id\"]\ntrain_encoded = train_encoded.drop(columns=[c for c in id_cols if c in train_encoded.columns])","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:16.583387Z","iopub.execute_input":"2026-02-12T16:08:16.583652Z","iopub.status.idle":"2026-02-12T16:08:16.589682Z","shell.execute_reply.started":"2026-02-12T16:08:16.583621Z","shell.execute_reply":"2026-02-12T16:08:16.588932Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\nimport pandas as pd\nimport numpy as np\nfrom tabicl import TabICLRegressor\n\ntrain_df = train_encoded\ntest_df = test_encoded\ntest_ids = pd.read_csv(\"test_scaled.csv\")[\"row_id\"]\n\ntarget_col = \"total_bid\"\nid_cols = [\"row_id\", \"job_id\", \"contractor_id\"]\n\nX = train_df.drop(columns=[c for c in [target_col] + id_cols if c in train_df.columns])\ny = np.log1p(train_df[target_col].values)\n\nX_test = test_df.drop(columns=[c for c in [target_col] + id_cols if c in test_df.columns])\n\nX, X_test = X.align(X_test, join=\"left\", axis=1, fill_value=0)\n\nmodel = TabICLRegressor(n_estimators=64)\n\nmodel.fit(X, y)\n\ntest_preds_log = model.predict(X_test)\ntest_preds = np.expm1(test_preds_log)\ntest_preds = np.maximum(test_preds, 0)\n\nsubmission = pd.DataFrame({\n    \"row_id\": test_ids,\n    \"total_bid\": test_preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Saved: submission.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-12T16:08:16.590661Z","iopub.execute_input":"2026-02-12T16:08:16.590928Z","iopub.status.idle":"2026-02-12T16:09:17.882494Z","shell.execute_reply.started":"2026-02-12T16:08:16.590909Z","shell.execute_reply":"2026-02-12T16:09:17.881765Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}