{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.12.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":130062,"databundleVersionId":15606741,"sourceType":"competition"},{"sourceId":14766305,"sourceType":"datasetVersion","datasetId":9438580}],"dockerImageVersionId":31259,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"Loading and previewing the csvs....","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np\n\ntest_raw = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/raw_test.csv\")\ntrain_raw = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/raw_train.csv\")\ntest = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/test.csv\")\n\ntrain_summary = pd.read_csv(\"/kaggle/input/bidding-predictions-for-construction/train_summary.csv\")\n\n# run these at ur discretion to preview the datasets! :)\n# test_raw.head()\n# train_raw.head()\n# train_summary.head()\n# test.head()","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:04.987293Z","iopub.execute_input":"2026-02-08T01:38:04.988067Z","iopub.status.idle":"2026-02-08T01:38:07.465218Z","shell.execute_reply.started":"2026-02-08T01:38:04.988015Z","shell.execute_reply":"2026-02-08T01:38:07.464084Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def summarize_df(df, name, top_n=20):\n    print(f\"\\n=== {name} ===\")\n    print(f\"shape: {df.shape[0]} rows x {df.shape[1]} cols\\n\")\n    \n    # dtypes?\n    dtypes = df.dtypes.astype(str)\n    \n    # nans?\n    miss_count = df.isna().sum()\n    miss_pct = (miss_count / len(df)) * 100\n    \n    # unique counts\n    nunique = df.nunique(dropna=True)\n    \n    # build summary table\n    summary = pd.DataFrame({\n        \"dtype\": dtypes,\n        \"missing_count\": miss_count,\n        \"missing_pct\": miss_pct.round(3),\n        \"n_unique\": nunique\n    }).sort_values([\"missing_count\", \"n_unique\"], ascending=False)\n    \n    display(summary)\n    \n    # numeric stats\n    num_cols = df.select_dtypes(include=[np.number]).columns\n    if len(num_cols) > 0:\n        print(\"\\nNumeric describe (log-safe preview):\")\n        display(df[num_cols].describe().T)\n    \n    # categorical cardinality\n    cat_cols = df.select_dtypes(include=[\"object\", \"category\"]).columns\n    if len(cat_cols) > 0:\n        print(f\"\\nCategorical columns (top {top_n} by cardinality):\")\n        cat_card = nunique[cat_cols].sort_values(ascending=False)\n        display(cat_card.head(top_n))\n    \n    # duplicates\n    dup_count = df.duplicated().sum()\n    print(f\"\\nDuplicate rows: {dup_count}\")\n    \n    return summary\n\nsummary_train_summary = summarize_df(train_summary, \"train_summary\")\nsummary_train_raw = summarize_df(train_raw, \"train_raw\")\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:07.467180Z","iopub.execute_input":"2026-02-08T01:38:07.467557Z","iopub.status.idle":"2026-02-08T01:38:09.742868Z","shell.execute_reply.started":"2026-02-08T01:38:07.467525Z","shell.execute_reply":"2026-02-08T01:38:09.741823Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"dataset looks nicely organized. moving on...","metadata":{}},{"cell_type":"code","source":"train_summary[\"row_id\"] = train_summary[\"job_id\"] + \"__\" + train_summary[\"contractor_id\"]\ntest[\"row_id\"] = test[\"job_id\"] + \"__\" + test[\"contractor_id\"]\n\n\nfor df in [train_summary, test]:\n    df[\"bid_date\"] = pd.to_datetime(df[\"bid_date\"])\n    df[\"bid_year\"] = df[\"bid_date\"].dt.year\n    df[\"bid_month\"] = df[\"bid_date\"].dt.month\n    df[\"bid_day\"] = df[\"bid_date\"].dt.day\n    df[\"bid_dow\"] = df[\"bid_date\"].dt.dayofweek\n    df[\"bid_week\"] = df[\"bid_date\"].dt.isocalendar().week.astype(int)\n    df[\"bid_date_ordinal\"] = df[\"bid_date\"].map(pd.Timestamp.toordinal)\n\n\ndef build_raw_agg(raw_df):\n    agg_num = raw_df.groupby(\"row_id\").agg(\n        quantity_sum=(\"quantity\", \"sum\"),\n        quantity_mean=(\"quantity\", \"mean\"),\n        quantity_std=(\"quantity\", \"std\"),\n        quantity_min=(\"quantity\", \"min\"),\n        quantity_max=(\"quantity\", \"max\"),\n        num_pay_items_mean=(\"num_pay_items\", \"mean\"),\n        num_pay_items_max=(\"num_pay_items\", \"max\"),\n        num_pay_items_min=(\"num_pay_items\", \"min\"),\n    )\n\n\n    agg_cat = raw_df.groupby(\"row_id\").agg(\n        n_pay_item_id=(\"pay_item_id\", \"nunique\"),\n        n_pay_item_desc=(\"pay_item_description\", \"nunique\"),\n        n_unit_english=(\"unit_english_id\", \"nunique\"),\n        n_category_id=(\"category_id\", \"nunique\"),\n        n_category_desc=(\"category_description\", \"nunique\"),\n        n_job_category=(\"job_category_description\", \"nunique\"),\n    )\n\n\n    agg_cnt = raw_df.groupby(\"row_id\").size().to_frame(\"n_line_items\")\n\n\n    agg = pd.concat([agg_num, agg_cat, agg_cnt], axis=1)\n    return agg\n\ntrain_raw_agg = build_raw_agg(train_raw)\ntest_raw_agg = build_raw_agg(test_raw)\n\ntrain_merged = train_summary.merge(train_raw_agg, on=\"row_id\", how=\"left\")\ntest_merged = test.merge(test_raw_agg, on=\"row_id\", how=\"left\")\n\ntrain_merged.to_csv(\"my_train_dataset.csv\", index=False)\ntest_merged.to_csv(\"my_test_dataset.csv\", index=False)\n\nprint(\"saved: my_train_dataset.csv and my_test_dataset.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:09.743988Z","iopub.execute_input":"2026-02-08T01:38:09.744292Z","iopub.status.idle":"2026-02-08T01:38:10.949532Z","shell.execute_reply.started":"2026-02-08T01:38:09.744264Z","shell.execute_reply":"2026-02-08T01:38:10.948562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_merged.head(15)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:10.951851Z","iopub.execute_input":"2026-02-08T01:38:10.952153Z","iopub.status.idle":"2026-02-08T01:38:10.974571Z","shell.execute_reply.started":"2026-02-08T01:38:10.952125Z","shell.execute_reply":"2026-02-08T01:38:10.973611Z"}},"outputs":[],"execution_count":null},{"cell_type":"markdown","source":"Scaling:\nnumeric features: z score\ntemporal features: transformed into \n    - relative day (day / number of days in entire time period that the dataset encompasses)\n    - relative day in year (day / total number of days in that year) \n    - relative day in month (day / total number of days in that month)","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\ninflation = pd.read_csv(\"/kaggle/input/inflation/inflation.csv\")\ninflation[\"observation_date\"] = pd.to_datetime(inflation[\"observation_date\"])\ninflation[\"year\"] = inflation[\"observation_date\"].dt.year\ninflation[\"month\"] = inflation[\"observation_date\"].dt.month\ninflation = inflation.rename(columns={\"CPIAUCSL\": \"inflation_index\"})\n\ntrain_scaled = train_merged.copy()\ntest_scaled = test_merged.copy()\n\ntrain_scaled = train_scaled.merge(\n    inflation[[\"year\",\"month\",\"inflation_index\"]],\n    left_on=[\"bid_year\",\"bid_month\"],\n    right_on=[\"year\",\"month\"],\n    how=\"left\"\n).drop(columns=[\"year\",\"month\"])\n\ntest_scaled = test_scaled.merge(\n    inflation[[\"year\",\"month\",\"inflation_index\"]],\n    left_on=[\"bid_year\",\"bid_month\"],\n    right_on=[\"year\",\"month\"],\n    how=\"left\"\n).drop(columns=[\"year\",\"month\"])\n\ntrain_scaled[\"inflation_index\"] = train_scaled[\"inflation_index\"].ffill().bfill()\ntest_scaled[\"inflation_index\"] = test_scaled[\"inflation_index\"].ffill().bfill()\n\ntrain_scaled[\"bid_date\"] = pd.to_datetime(train_scaled[\"bid_date\"])\ntest_scaled[\"bid_date\"] = pd.to_datetime(test_scaled[\"bid_date\"])\n\nmin_date = train_scaled[\"bid_date\"].min()\nmax_date = train_scaled[\"bid_date\"].max()\ndate_range_days = max((max_date - min_date).days, 1)\n\nfor df in [train_scaled, test_scaled]:\n    df[\"day_relative_to_full_time_period\"] = (\n        (df[\"bid_date\"] - min_date).dt.days / date_range_days\n    ).clip(0, 1)\n\n    day_of_year = df[\"bid_date\"].dt.dayofyear\n    days_in_year = df[\"bid_date\"].dt.is_leap_year.astype(int).map({1: 366, 0: 365})\n    df[\"day_relative_to_full_year\"] = day_of_year / days_in_year\n\n    df[\"day_relative_to_month\"] = df[\"bid_date\"].dt.day / df[\"bid_date\"].dt.days_in_month\n    df[\"day_relative_to_week\"] = df[\"bid_date\"].dt.dayofweek / 6\n\ndate_cols_to_drop = [\n    \"bid_date\", \"bid_year\", \"bid_month\", \"bid_day\", \"bid_dow\", \"bid_week\", \"bid_date_ordinal\"\n]\nfor col in date_cols_to_drop:\n    if col in train_scaled.columns:\n        train_scaled.drop(columns=col, inplace=True)\n    if col in test_scaled.columns:\n        test_scaled.drop(columns=col, inplace=True)\n\ndate_ratio_cols = [\n    \"day_relative_to_full_time_period\",\n    \"day_relative_to_full_year\",\n    \"day_relative_to_month\",\n    \"day_relative_to_week\"\n]\n\ntarget_col = \"total_bid\"\nexclude_cols = set(date_ratio_cols + [target_col])\n\nnum_cols = train_scaled.select_dtypes(include=[np.number]).columns\nnum_cols = [c for c in num_cols if c not in exclude_cols]\n\nfor col in num_cols:\n    train_scaled[col] = np.log1p(train_scaled[col].clip(lower=0))\n    test_scaled[col] = np.log1p(test_scaled[col].clip(lower=0))\n\ntrain_scaled.to_csv(\"train_scaled.csv\", index=False)\ntest_scaled.to_csv(\"test_scaled.csv\", index=False)\n\nprint(\"Saved: train_scaled.csv and test_scaled.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:10.975890Z","iopub.execute_input":"2026-02-08T01:38:10.976291Z","iopub.status.idle":"2026-02-08T01:38:11.497538Z","shell.execute_reply.started":"2026-02-08T01:38:10.976261Z","shell.execute_reply":"2026-02-08T01:38:11.496760Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_scaled.head(20)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:11.498913Z","iopub.execute_input":"2026-02-08T01:38:11.499240Z","iopub.status.idle":"2026-02-08T01:38:11.530260Z","shell.execute_reply.started":"2026-02-08T01:38:11.499213Z","shell.execute_reply":"2026-02-08T01:38:11.529434Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"train_scaled = pd.read_csv(\"train_scaled.csv\")\ntest_scaled = pd.read_csv(\"test_scaled.csv\")\n\nhigh_card_cols = [\"job_id\", \"contractor_id\", \"primary_location\"]\nfor col in high_card_cols:\n    if col in train_scaled.columns:\n        all_vals = pd.concat([train_scaled[col], test_scaled[col]], axis=0).astype(str)\n        codes, uniques = pd.factorize(all_vals, sort=True)\n        train_scaled[col] = codes[:len(train_scaled)]\n        test_scaled[col] = codes[len(train_scaled):]\n\nlow_card_cols = [\"job_category_description\"]\ntrain_encoded = pd.get_dummies(train_scaled, columns=low_card_cols, drop_first=False)\ntest_encoded = pd.get_dummies(test_scaled, columns=low_card_cols, drop_first=False)\n\ntrain_encoded, test_encoded = train_encoded.align(\n    test_encoded, join=\"left\", axis=1, fill_value=0\n)\n\ntarget_col = \"total_bid\"\nif target_col not in train_encoded.columns and target_col in train_scaled.columns:\n    train_encoded[target_col] = train_scaled[target_col]\n\nbool_cols = train_encoded.select_dtypes(include=[\"bool\"]).columns\ntrain_encoded[bool_cols] = train_encoded[bool_cols].astype(int)\n\nbool_cols_test = test_encoded.select_dtypes(include=[\"bool\"]).columns\ntest_encoded[bool_cols_test] = test_encoded[bool_cols_test].astype(int)\n\ntrain_encoded.to_csv(\"train_encoded.csv\", index=False)\ntest_encoded.to_csv(\"test_encoded.csv\", index=False)\n\nprint(\"Saved: train_encoded.csv and test_encoded.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:11.531423Z","iopub.execute_input":"2026-02-08T01:38:11.531776Z","iopub.status.idle":"2026-02-08T01:38:12.091311Z","shell.execute_reply.started":"2026-02-08T01:38:11.531708Z","shell.execute_reply":"2026-02-08T01:38:12.090316Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"id_cols = [\"row_id\", \"job_id\", \"contractor_id\"]\ntrain_encoded = train_encoded.drop(columns=[c for c in id_cols if c in train_encoded.columns])\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:12.092532Z","iopub.execute_input":"2026-02-08T01:38:12.093387Z","iopub.status.idle":"2026-02-08T01:38:12.102172Z","shell.execute_reply.started":"2026-02-08T01:38:12.093343Z","shell.execute_reply":"2026-02-08T01:38:12.101115Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T01:38:12.103420Z","iopub.execute_input":"2026-02-08T01:38:12.103788Z","iopub.status.idle":"2026-02-08T01:39:02.097128Z","shell.execute_reply.started":"2026-02-08T01:38:12.103748Z","shell.execute_reply":"2026-02-08T01:39:02.095507Z"}},"outputs":[],"execution_count":null},{"cell_type":"raw","source":"","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport lightgbm as lgb\nfrom sklearn.model_selection import GroupKFold\n\ntrain_df = train_encoded\ntest_df = test_encoded\ntest_ids = pd.read_csv(\"test_scaled.csv\")[\"row_id\"]\n\ntarget_col = \"total_bid\"\nid_cols = [\"row_id\", \"job_id\", \"contractor_id\"]\n\nX = train_df.drop(columns=[c for c in [target_col] + id_cols if c in train_df.columns])\ny = np.log1p(train_df[target_col].values)\n\nX_test = test_df.drop(columns=[c for c in [target_col] + id_cols if c in test_df.columns])\n\nX, X_test = X.align(X_test, join=\"left\", axis=1, fill_value=0)\n\ngroups = pd.read_csv(\"train_scaled.csv\")[\"job_id\"].values\n\ndef rmsle_eval(preds, dataset):\n    y_true = dataset.get_label()\n    return \"rmse_log\", np.sqrt(np.mean((preds - y_true) ** 2)), False\n\nparams = {\n    \"objective\": \"regression\",\n    \"metric\": \"None\",\n    \"learning_rate\": 0.00485,\n    \"num_leaves\": 32,\n    \"max_depth\":14,\n    \"feature_fraction\": 0.6,\n    \"min_data_in_leaf\": 21,\n    \"bagging_fraction\": 0.57,\n    \"bagging_freq\": 1,\n    \"lambda_l1\": 0.0095,\n    \"lambda_l2\": 7.4,\n    \"min_gain_to_split\": 0.055,\n    \"seed\": 42,\n    \"verbosity\": -1\n}\n\ngkf = GroupKFold(n_splits=5)\nfold_scores = []\nbest_iters = []\n\nfor fold, (tr_idx, va_idx) in enumerate(gkf.split(X, y=None, groups=groups), 1):\n    X_tr, X_va = X.iloc[tr_idx], X.iloc[va_idx]\n    y_tr, y_va = y[tr_idx], y[va_idx]\n\n    dtrain = lgb.Dataset(X_tr, label=y_tr)\n    dvalid = lgb.Dataset(X_va, label=y_va)\n\n    model = lgb.train(\n        params,\n        dtrain,\n        valid_sets=[dvalid],\n        feval=rmsle_eval,\n        num_boost_round=50000,\n        callbacks=[\n            lgb.early_stopping(300, verbose=True),\n            lgb.log_evaluation(100)\n        ]\n    )\n\n    preds_log = model.predict(X_va, num_iteration=model.best_iteration)\n    preds_nominal = np.expm1(preds_log)\n    y_va_nominal = np.expm1(y_va)\n\n    rmsle = np.sqrt(np.mean((np.log1p(preds_nominal) - np.log1p(y_va_nominal))**2))\n    fold_scores.append(rmsle)\n    best_iters.append(model.best_iteration)\n\n    print(f\"Fold {fold} RMSLE: {rmsle:.6f}\")\n\nprint(f\"Mean RMSLE: {np.mean(fold_scores):.6f} ± {np.std(fold_scores):.6f}\")\n\nbest_iter = int(np.mean(best_iters))\n\ndtrain_full = lgb.Dataset(X, label=y)\nfinal_model = lgb.train(\n    params,\n    dtrain_full,\n    feval=rmsle_eval,\n    num_boost_round=best_iter\n)\n\ntest_preds_log = final_model.predict(X_test)\ntest_preds = np.expm1(test_preds_log)\ntest_preds = np.maximum(test_preds, 0)\n\nsubmission = pd.DataFrame({\n    \"row_id\": test_ids,\n    \"total_bid\": test_preds\n})\n\nsubmission.to_csv(\"submission.csv\", index=False)\nprint(\"Saved: submission.csv\")\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2026-02-08T15:09:51.054703Z","iopub.execute_input":"2026-02-08T15:09:51.055073Z","iopub.status.idle":"2026-02-08T15:09:55.609442Z","shell.execute_reply.started":"2026-02-08T15:09:51.055040Z","shell.execute_reply":"2026-02-08T15:09:55.607521Z"}},"outputs":[],"execution_count":null}]}