{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.17","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"tpuV5e8","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"},{"sourceId":105113547,"sourceType":"kernelVersion"}],"dockerImageVersionId":31040,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:38:43.650047Z","iopub.execute_input":"2025-10-09T04:38:43.650299Z","iopub.status.idle":"2025-10-09T04:38:43.665672Z","shell.execute_reply.started":"2025-10-09T04:38:43.650283Z","shell.execute_reply":"2025-10-09T04:38:43.662700Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\nprint(os.listdir(\"/kaggle/input\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:38:47.343078Z","iopub.execute_input":"2025-10-09T04:38:47.343308Z","iopub.status.idle":"2025-10-09T04:38:47.350564Z","shell.execute_reply.started":"2025-10-09T04:38:47.343290Z","shell.execute_reply":"2025-10-09T04:38:47.347551Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import os\n\n# Let's list what's inside the avazu dataset folder\nprint(\"Files inside avazu-ctr-prediction:\")\nprint(os.listdir(\"/kaggle/input/avazu-ctr-prediction\"))\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:38:49.854791Z","iopub.execute_input":"2025-10-09T04:38:49.855081Z","iopub.status.idle":"2025-10-09T04:38:49.867885Z","shell.execute_reply.started":"2025-10-09T04:38:49.855051Z","shell.execute_reply":"2025-10-09T04:38:49.864172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nimport joblib\nfrom pathlib import Path\n\n# Paths\nRAW_PATH = Path(\"/kaggle/input/avazu-ctr-prediction/train.gz\")\nPROC_PATH = Path(\"/kaggle/working/train_processed.parquet\")\nENCODERS_PATH = Path(\"/kaggle/working/encoders.joblib\")\nSCALER_PATH = Path(\"/kaggle/working/scaler.joblib\")\n\n# Parameters\nRANDOM_STATE = 42\nSAMPLE_ROWS = None  # set to an integer for faster testing\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:38:51.567642Z","iopub.execute_input":"2025-10-09T04:38:51.567886Z","iopub.status.idle":"2025-10-09T04:38:51.576462Z","shell.execute_reply.started":"2025-10-09T04:38:51.567866Z","shell.execute_reply":"2025-10-09T04:38:51.573086Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Load compressed train file\ndf = pd.read_csv(RAW_PATH, nrows=SAMPLE_ROWS, compression='gzip')\nprint(\"Train shape:\", df.shape)\ndf.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:38:53.670374Z","iopub.execute_input":"2025-10-09T04:38:53.670597Z","iopub.status.idle":"2025-10-09T04:40:06.287583Z","shell.execute_reply.started":"2025-10-09T04:38:53.670580Z","shell.execute_reply":"2025-10-09T04:40:06.283088Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Fill missing values\ndf.fillna(\"unknown\", inplace=True)\n\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:40:32.084019Z","iopub.execute_input":"2025-10-09T04:40:32.084261Z","iopub.status.idle":"2025-10-09T04:40:40.747448Z","shell.execute_reply.started":"2025-10-09T04:40:32.084243Z","shell.execute_reply":"2025-10-09T04:40:40.743564Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"\n\n# Target column\ntarget_col = 'click'\n\n# Time column\ntime_col = 'hour'\n\n# User/Ad columns for aggregates\nad_col = 'id'\nuser_col = 'device_id'  # Using device_id as user identifier\nsession_col = None  # Not available\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:47:00.711734Z","iopub.execute_input":"2025-10-09T04:47:00.711978Z","iopub.status.idle":"2025-10-09T04:47:00.719892Z","shell.execute_reply.started":"2025-10-09T04:47:00.711960Z","shell.execute_reply":"2025-10-09T04:47:00.716067Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Convert Avazu 'hour' column to datetime\ndf['hour'] = df['hour'].astype(str)\ndf['datetime'] = pd.to_datetime(df['hour'], format='%y%m%d%H', errors='coerce')\n\n# Extract features\ndf['hour_of_day'] = df['datetime'].dt.hour\ndf['day_of_week'] = df['datetime'].dt.weekday\ndf['is_weekend'] = df['day_of_week'].isin([5,6]).astype('int')\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:47:14.794333Z","iopub.execute_input":"2025-10-09T04:47:14.794574Z","iopub.status.idle":"2025-10-09T04:47:25.977494Z","shell.execute_reply.started":"2025-10-09T04:47:14.794553Z","shell.execute_reply":"2025-10-09T04:47:25.974163Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Global click rate\nglobal_ctr = df[target_col].mean()\n\n# Ad popularity (CTR per ad)\nad_stats = df.groupby(ad_col)[target_col].agg(['sum','count']).rename(columns={'sum':'ad_clicks','count':'ad_impr'})\nad_stats['ad_ctr'] = ad_stats['ad_clicks'] / (ad_stats['ad_impr'] + 1e-9)\nad_map = ad_stats['ad_ctr'].to_dict()\ndf['ad_popularity'] = df[ad_col].map(ad_map).fillna(global_ctr)\n\n# User past CTR (device_id)\nuser_stats = df.groupby(user_col)[target_col].agg(['sum','count']).rename(columns={'sum':'user_clicks','count':'user_impr'})\nuser_stats['user_ctr'] = user_stats['user_clicks'] / (user_stats['user_impr'] + 1e-9)\nuser_map = user_stats['user_ctr'].to_dict()\ndf['user_past_ctr'] = df[user_col].map(user_map).fillna(global_ctr)\n\n# Session length (not available, set default 1)\ndf['session_len'] = 1\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:47:45.254774Z","iopub.execute_input":"2025-10-09T04:47:45.255001Z","iopub.status.idle":"2025-10-09T04:48:43.168931Z","shell.execute_reply.started":"2025-10-09T04:47:45.254983Z","shell.execute_reply":"2025-10-09T04:48:43.163807Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Categorical columns\ncat_cols = [\n    'C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n    'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n    'device_model', 'device_type', 'device_conn_type',\n    'C14','C15','C16','C17','C18','C19','C20','C21'\n]\n\n# Fit label encoders\ncat_encoders = {}\nfor col in cat_cols:\n    le = LabelEncoder()\n    df[col] = le.fit_transform(df[col].astype(str))\n    cat_encoders[col] = le\n\n# Save encoders\njoblib.dump(cat_encoders, ENCODERS_PATH)\nprint(\"Saved categorical encoders to:\", ENCODERS_PATH)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:50:49.102574Z","iopub.execute_input":"2025-10-09T04:50:49.102806Z","iopub.status.idle":"2025-10-09T04:54:24.041823Z","shell.execute_reply.started":"2025-10-09T04:50:49.102788Z","shell.execute_reply":"2025-10-09T04:54:24.036626Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Numeric features\nnum_cols = ['ad_popularity','user_past_ctr','session_len','hour_of_day','day_of_week']\n\nscaler = StandardScaler()\ndf[num_cols] = scaler.fit_transform(df[num_cols])\n\n# Save scaler\njoblib.dump(scaler, SCALER_PATH)\nprint(\"Saved scaler to:\", SCALER_PATH)\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:54:37.928133Z","iopub.execute_input":"2025-10-09T04:54:37.928377Z","iopub.status.idle":"2025-10-09T04:54:39.935706Z","shell.execute_reply.started":"2025-10-09T04:54:37.928359Z","shell.execute_reply":"2025-10-09T04:54:39.791274Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"# Save processed train dataset (Parquet for speed & size)\ndf.to_parquet(PROC_PATH, index=False)\nprint(\"Saved processed train dataset to:\", PROC_PATH)\ndf.head()\n","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-10-09T04:54:49.346789Z","iopub.execute_input":"2025-10-09T04:54:49.347010Z","iopub.status.idle":"2025-10-09T04:55:11.123924Z","shell.execute_reply.started":"2025-10-09T04:54:49.346993Z","shell.execute_reply":"2025-10-09T04:55:11.119697Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"","metadata":{"trusted":true},"outputs":[],"execution_count":null}]}