{"metadata":{"interpreter":{"hash":"0e24798decccf32fc49541ef3e02f081f328b9dde94a829d0c16468fee7fb2ac"},"kernelspec":{"name":"python3","display_name":"Python 3","language":"python"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30761,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install --upgrade scikit-learn","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:13:47.630332Z","iopub.execute_input":"2024-09-10T12:13:47.630816Z","iopub.status.idle":"2024-09-10T12:14:12.343555Z","shell.execute_reply.started":"2024-09-10T12:13:47.630766Z","shell.execute_reply":"2024-09-10T12:14:12.342008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nfrom matplotlib import pyplot as plt\nimport seaborn as sns\nfrom scipy import stats, signal\nfrom sklearn.preprocessing import TargetEncoder\n\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.model_selection import GridSearchCV","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:14:21.810535Z","iopub.execute_input":"2024-09-10T12:14:21.811847Z","iopub.status.idle":"2024-09-10T12:14:21.820872Z","shell.execute_reply.started":"2024-09-10T12:14:21.811775Z","shell.execute_reply":"2024-09-10T12:14:21.818811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load Data\nDue to the limited memory, we sample 5% data instead.","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('/kaggle/input/avazu-ctr-prediction/train.gz', compression='gzip', parse_dates=['hour'], date_format='%y%m%d%H')\ntrain_df.drop(columns=['id'], inplace=True)\n\n# add hour and weakday feature, because the range of the dates only across 10 days\ntrain_df['weekday'] = train_df.hour.dt.weekday\ntrain_df['hour'] = train_df.hour.dt.hour\n\ntrain_df = train_df.sample(frac=0.05, random_state=42)","metadata":{"execution":{"iopub.status.busy":"2024-09-10T12:14:45.195947Z","iopub.execute_input":"2024-09-10T12:14:45.197256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"check null values","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"check the percantage of uniqueness","metadata":{}},{"cell_type":"code","source":"uni_df = train_df.nunique().to_frame(name='cnt')\nuni_df['percentage'] = np.round(uni_df / len(train_df), 4)\nuni_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Visualization","metadata":{}},{"cell_type":"code","source":"sns.heatmap(\n    train_df\n    .groupby(['hour', 'banner_pos'])\n    .click.mean()\n    .sort_values(ascending=False)\n    .reset_index()\n    .pivot(index='hour', columns='banner_pos', values='click')\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"CTR under certain banner position are stable. For `banner_pos == 7`, CTRs in night are relatively high.","metadata":{}},{"cell_type":"code","source":"sns.heatmap(\n    train_df\n    .groupby(['site_category', 'app_category'])\n    .click.mean()\n    .sort_values(ascending=False)\n    .reset_index()\n    .pivot(index='site_category', columns='app_category', values='click')\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Business insights may lie in what the sites and apps are. And why some combinations have high(low) CTR.","metadata":{}},{"cell_type":"markdown","source":"# Feature Engineering\nTo get rid of the curse of the dimensionality, dive into each features to see if there is any pattern.","metadata":{}},{"cell_type":"markdown","source":"There are not lots of unique values in all of the columns. Thus, we assume that all features are categorical variables.   \nHowever, `device_id` and `device_ip` are quite diverse.","metadata":{}},{"cell_type":"code","source":"X_train = train_df.drop(columns=['click'])\ny_train = train_df.click\nenc_auto = TargetEncoder(smooth=\"auto\")\nX_trans = enc_auto.fit_transform(X_train, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Use target encoder to handle with categorical variables with high cardinality. Avoid high-dimensionality using one-hot-encoding.","metadata":{}},{"cell_type":"code","source":"fig, axes = plt.subplots(6, 4, figsize=(30, 36))\n\nfor i, feature in enumerate(enc_auto.feature_names_in_):\n    nums = np.unique(X_trans[:, i])\n    r = np.arange(nums.min(), nums.max(), 0.01)\n    sns.histplot(data=nums, kde=True, ax=axes[i // 4, i % 4], bins=10)\n    axes[i // 4, i % 4].set_title(feature)\n    kernel = stats.gaussian_kde(nums)\n    curve = kernel(r)\n    peaks, _ = signal.find_peaks(curve)\n    modal_points = r[peaks]\n    for p in modal_points:\n        axes[i // 4, i % 4].axvline(p, color='black')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that some features reaches 1.0, which means samples under this category will certainly click. It's a good sign to investigate the features.","metadata":{}},{"cell_type":"markdown","source":"# Feature Selection","metadata":{}},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nX_pre, X_val, y_pre, y_val = train_test_split(\n    X_train,\n    y_train,\n    test_size=0.5,\n    shuffle=False,\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Demonstrate the feature importance of target encoded features.  \nTo avoid the overfitting problem, it's fair to dive into the unseen features encoded by preprocessing data (pre). We use X_pre to encode X_val and verify the feature importance on (X_val, y_val).\n\n","metadata":{}},{"cell_type":"code","source":"enc_imp = TargetEncoder(smooth=\"auto\")\nX_train_trans = enc_imp.fit_transform(X_pre, y_pre)\nX_val_trans = enc_imp.transform(X_val)\n\nforest = RandomForestClassifier(n_estimators=100, max_depth=10, random_state=42, n_jobs=-1, verbose=2)\nforest.fit(X_val_trans, y_val)\nimportances = forest.feature_importances_","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"forest_importances = pd.Series(importances, index=enc_auto.feature_names_in_).sort_values(ascending=False)\n\nfig, ax = plt.subplots()\nforest_importances.plot.bar(ax=ax)\n\nax.set_title(\"Feature importances using MDI\")\nax.set_ylabel(\"Mean decrease in impurity\")\nfig.tight_layout()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Note that although `device_ip` and `device_id` are diverse, they do show strong or moderate feature importance. Therefore, we could consider keep them. Also since # features << # samples, I tend to preserve all features.","metadata":{}},{"cell_type":"markdown","source":"# Hyperparameter Tuning\nWe choose Random Forest as the backbone model, and tune the hyperparameter of `n_estimators` and `max_depth` by CV.  \nSince the competition has already defined the evalution metirc, we choose `log loss` as the criterion.","metadata":{}},{"cell_type":"code","source":"param_grid = {\n    'max_depth': [10, 100, 300],\n    'n_estimators': [10, 50, 200],\n}\n\nclf = RandomForestClassifier(criterion='log_loss', class_weight=\"balanced\")\n\n# Create the GridSearchCV object\n# where the score is maximized\ngrid_search = GridSearchCV(clf, param_grid, cv=5, scoring='neg_log_loss', refit=False, n_jobs=-1, verbose=2)\n\n# Fit the GridSearchCV object to the training data\ngrid_search.fit(X_trans, y_train)\n\nprint(\"Best set of hyperparameters: \", grid_search.best_params_)\nprint(\"Best score: \", grid_search.best_score_)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The best setting of the hyperparameters is `{'max_depth': 10, 'n_estimators': 200}`.","metadata":{}},{"cell_type":"markdown","source":"# Final Predict\n\nTrain the whole training dataset with tuned hyperparameter","metadata":{}},{"cell_type":"code","source":"clf = RandomForestClassifier(**grid_search.best_params_, n_jobs=-1, criterion='log_loss', class_weight=\"balanced\")\nclf.fit(X_trans, y_train)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Prepare the test dataset and apply prediction.","metadata":{}},{"cell_type":"code","source":"test_df = pd.read_csv('/kaggle/input/avazu-ctr-prediction/test.gz, compression='gzip', parse_dates=['hour'], date_format='%y%m%d%H')\ntest_df.drop(columns=['id'], inplace=True)\n\ntest_df['weekday'] = test_df.hour.dt.weekday\ntest_df['hour'] = test_df.hour.dt.hour\n\nX_test = enc_auto.transform(test_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.read_csv('/kaggle/input/avazu-ctr-prediction/sampleSubmission.gz', compression='gzip')\nsubmission['click'] = clf.predict_proba(X_test)[:, 1]\nsubmission.to_csv('submission.csv', index = False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"The private score is 0.64867 and the pubic one is 0.64900. To enhance the model performance, we can consider:\n- Increase the training dataset. Note that we only use 5% data.\n- Consider more features, e.g., user behavioral data, embedding of the ads, time series (what's the performance N hours ago)\n- Use strong models, e.g., the increase of n_estimatiors, boosting models (Xgboost, Catboost), etc.","metadata":{}}]}