{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-07T08:13:30.591427Z","iopub.execute_input":"2022-09-07T08:13:30.592583Z","iopub.status.idle":"2022-09-07T08:13:30.602069Z","shell.execute_reply.started":"2022-09-07T08:13:30.592519Z","shell.execute_reply":"2022-09-07T08:13:30.600782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Quick Intro\nIn this task, we want to try to predict click through rate (CTR) which is a very important metrid for evaluatiing advertising performance. We will use 11 days worth of Azazu data to build ad test our models (10 days for training and 1 day for testing).","metadata":{}},{"cell_type":"code","source":"# import necessary libraries\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom category_encoders import TargetEncoder\n","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:13:35.551614Z","iopub.execute_input":"2022-09-07T08:13:35.552551Z","iopub.status.idle":"2022-09-07T08:13:37.896109Z","shell.execute_reply.started":"2022-09-07T08:13:35.552492Z","shell.execute_reply":"2022-09-07T08:13:37.895175Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data paths\ntrain_data_path = \"/kaggle/input/avazu-ctr-prediction/train.gz\"\ntest_data_path = \"/kaggle/input/avazu-ctr-prediction/test.gz\"","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:13:40.816270Z","iopub.execute_input":"2022-09-07T08:13:40.816659Z","iopub.status.idle":"2022-09-07T08:13:40.821265Z","shell.execute_reply.started":"2022-09-07T08:13:40.816618Z","shell.execute_reply":"2022-09-07T08:13:40.820092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get train data sample using pd chunks. Each chunk has a size of 1000000\ntrain_data = pd.read_csv(train_data_path, chunksize = 1000000)\ntrain_df = pd.DataFrame()\n\nfor chunk in train_data:\n    train_df = pd.concat([train_df, chunk.sample(frac=.1, replace=False, random_state=123)], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:13:43.638337Z","iopub.execute_input":"2022-09-07T08:13:43.639132Z","iopub.status.idle":"2022-09-07T08:17:59.835360Z","shell.execute_reply.started":"2022-09-07T08:13:43.639089Z","shell.execute_reply":"2022-09-07T08:17:59.833512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# get the test data\ntest_data = pd.read_csv(test_data_path, chunksize = 1000000)\ntest_df = pd.DataFrame()\n\nfor chunk in test_data:\n    test_df = pd.concat([test_df, chunk], axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:22:59.618123Z","iopub.execute_input":"2022-09-07T08:22:59.618808Z","iopub.status.idle":"2022-09-07T08:23:22.656517Z","shell.execute_reply.started":"2022-09-07T08:22:59.618763Z","shell.execute_reply":"2022-09-07T08:23:22.655198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df.loc[0:4042896]","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:23:36.973057Z","iopub.execute_input":"2022-09-07T08:23:36.974055Z","iopub.status.idle":"2022-09-07T08:23:36.980053Z","shell.execute_reply.started":"2022-09-07T08:23:36.974004Z","shell.execute_reply":"2022-09-07T08:23:36.978673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check out the data\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:23:41.012490Z","iopub.execute_input":"2022-09-07T08:23:41.012807Z","iopub.status.idle":"2022-09-07T08:23:41.058929Z","shell.execute_reply.started":"2022-09-07T08:23:41.012776Z","shell.execute_reply":"2022-09-07T08:23:41.058037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#quick information on the data\ntrain_df.info()","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:23:48.568805Z","iopub.execute_input":"2022-09-07T08:23:48.569172Z","iopub.status.idle":"2022-09-07T08:23:48.604656Z","shell.execute_reply.started":"2022-09-07T08:23:48.569129Z","shell.execute_reply":"2022-09-07T08:23:48.603665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for i in train_df.columns:\n    print(train_df[i].value_counts())","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:23:52.217599Z","iopub.execute_input":"2022-09-07T08:23:52.217954Z","iopub.status.idle":"2022-09-07T08:24:01.866956Z","shell.execute_reply.started":"2022-09-07T08:23:52.217917Z","shell.execute_reply":"2022-09-07T08:24:01.865926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# some feature engineering and pipeline building\n# get the day of the weeko through a pipeline class\nclass DateTransformer(BaseEstimator, TransformerMixin):\n    '''Custom transformer for making date variables'''\n    def __init__(self, data_col):\n        self.data_col = data_col\n        \n    def build_date(self, hour):\n        day = str(hour)[4:6]\n        month = str(hour)[2:4]\n        year = '20'+str(hour)[0:2]\n        return day + '-' + month + '-' + year\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, X, y = None):\n        X_ = X.copy()\n        X_['fdate'] = pd.to_datetime(X_.hour.apply(self.build_date))\n        X_['weekday'] = X_['fdate'].dt.dayofweek.astype(str)\n        return X_\n\n# many of our categorical variables have high cardinality. let's try target encoding/mean encoding on them\nclass EncoderTransformer(BaseEstimator, TransformerMixin):\n    '''Custom transformer for encoding categorical variables'''\n    def __init__(self, encoder, target, train):\n        self.encoder = encoder\n        self.target = target\n        self.train = train\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, X, y=None):\n        X_ = X.copy()\n        if self.train == 1:\n            X_ = self.encoder.fit_transform(X_, self.target)\n        elif self.train == 0:\n            X_ = self.encoder.transform(X_, self.target)\n        return X_","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:24:14.661689Z","iopub.execute_input":"2022-09-07T08:24:14.661999Z","iopub.status.idle":"2022-09-07T08:24:14.674133Z","shell.execute_reply.started":"2022-09-07T08:24:14.661967Z","shell.execute_reply":"2022-09-07T08:24:14.673276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cat_cols = ['hour','C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n       'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n       'device_model', 'device_type', 'device_conn_type', 'C14', 'C15', 'C16',\n       'C17', 'C18', 'C19', 'C20', 'C21']","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:24:22.919271Z","iopub.execute_input":"2022-09-07T08:24:22.919597Z","iopub.status.idle":"2022-09-07T08:24:22.924693Z","shell.execute_reply.started":"2022-09-07T08:24:22.919562Z","shell.execute_reply":"2022-09-07T08:24:22.923791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# declare target encoder\nencoder = TargetEncoder(cols = ['C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n       'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n       'device_model', 'device_type', 'device_conn_type', 'C14', 'C15', 'C16',\n       'C17', 'C18', 'C19', 'C20', 'C21'])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:24:28.149485Z","iopub.execute_input":"2022-09-07T08:24:28.150535Z","iopub.status.idle":"2022-09-07T08:24:28.157336Z","shell.execute_reply.started":"2022-09-07T08:24:28.150423Z","shell.execute_reply":"2022-09-07T08:24:28.155649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make training and test sets\nX_train = train_df.drop(['click', 'id'], axis=1)\ny_train = train_df.click.values\nX_test = test_df.drop('id', axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:25:27.579515Z","iopub.execute_input":"2022-09-07T08:25:27.580160Z","iopub.status.idle":"2022-09-07T08:25:30.497382Z","shell.execute_reply.started":"2022-09-07T08:25:27.580117Z","shell.execute_reply":"2022-09-07T08:25:30.496293Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = ['C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n       'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n       'device_model', 'device_type', 'device_conn_type', 'C14', 'C15', 'C16',\n       'C17', 'C18', 'C19', 'C20', 'C21']","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:25:38.905470Z","iopub.execute_input":"2022-09-07T08:25:38.905822Z","iopub.status.idle":"2022-09-07T08:25:38.911571Z","shell.execute_reply.started":"2022-09-07T08:25:38.905783Z","shell.execute_reply":"2022-09-07T08:25:38.910600Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create pipeline steps\ntrain = 1\ncat_pipeline = Pipeline(steps=[\n    ('date_maker', DateTransformer('hour')),\n    ('en_coder', EncoderTransformer(encoder, y_train, train))\n    #('std_scaler', StandardScaler())\n])","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:25:42.816608Z","iopub.execute_input":"2022-09-07T08:25:42.817706Z","iopub.status.idle":"2022-09-07T08:25:42.824925Z","shell.execute_reply.started":"2022-09-07T08:25:42.817643Z","shell.execute_reply":"2022-09-07T08:25:42.823435Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create second pipeline step. alternatively we could have combined the two pipelines using the Column Transformer module\nnum_pipeline = Pipeline(steps=[\n    ('std_scaler', StandardScaler())\n])","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:25:48.717698Z","iopub.execute_input":"2022-09-07T08:25:48.718051Z","iopub.status.idle":"2022-09-07T08:25:48.724499Z","shell.execute_reply.started":"2022-09-07T08:25:48.718011Z","shell.execute_reply":"2022-09-07T08:25:48.722594Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit and transform our trainingv data on our pieplines\nX_train_tr = cat_pipeline.fit_transform(X_train)\nX_train_tr = num_pipeline.fit_transform(X_train_tr[num_cols])","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:25:52.437038Z","iopub.execute_input":"2022-09-07T08:25:52.437402Z","iopub.status.idle":"2022-09-07T08:27:35.798281Z","shell.execute_reply.started":"2022-09-07T08:25:52.437366Z","shell.execute_reply":"2022-09-07T08:27:35.797393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we want to use logistic regression because of the binary nature of our target. \n# but we have some class imbalance so let's first use a gird search to find the right weights for our model\nlr = LogisticRegression(solver='newton-cg')\n\n#Setting the range for class weights\nweights = np.linspace(0.0,0.99,6)\n\n\n#Creating a dictionary grid for grid search\nparam_grid = {'class_weight': [{0:x, 1:1.0-x} for x in weights]}\n\n#Fitting grid search to the train data with 5 folds\ngridsearch = GridSearchCV(estimator= lr, \n                          param_grid= param_grid,\n                          cv=StratifiedKFold(), \n                          n_jobs=-1, \n                          scoring='f1', \n                          verbose=2).fit(X_train_tr, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:28:44.529023Z","iopub.execute_input":"2022-09-07T08:28:44.529607Z","iopub.status.idle":"2022-09-07T08:42:33.183314Z","shell.execute_reply.started":"2022-09-07T08:28:44.529565Z","shell.execute_reply":"2022-09-07T08:42:33.181932Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# let's check out how the f1 score changes with different weight ratios\nsns.set_style('whitegrid')\nplt.figure(figsize=(12,8))\nweigh_data = pd.DataFrame({ 'score': gridsearch.cv_results_['mean_test_score'], 'weight': (1- weights)})\nsns.lineplot(weigh_data['weight'], weigh_data['score']);\nplt.xlabel('Weight for class 0')\nplt.ylabel('F1 score')\nplt.xticks([round(i/10,1) for i in range(0,11,1)])\nplt.title('Scoring for different class weights', fontsize=24)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:42:44.792988Z","iopub.execute_input":"2022-09-07T08:42:44.793360Z","iopub.status.idle":"2022-09-07T08:42:45.318133Z","shell.execute_reply.started":"2022-09-07T08:42:44.793317Z","shell.execute_reply":"2022-09-07T08:42:45.317297Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# declare sklearn logistic regressor and fit the model using the optimal classw weights from above\nlr = LogisticRegression(class_weight={0: 0.8, 1: 0.2}, max_iter=5000)\nlr.fit(X_train_tr, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:43:18.487763Z","iopub.execute_input":"2022-09-07T08:43:18.488134Z","iopub.status.idle":"2022-09-07T08:43:49.378330Z","shell.execute_reply.started":"2022-09-07T08:43:18.488097Z","shell.execute_reply":"2022-09-07T08:43:49.377193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# transform the test set using the same pipeline\ntrain = 0\ncat_pipeline = Pipeline(steps=[\n    ('date_maker', DateTransformer('hour')),\n    ('en_coder', EncoderTransformer(encoder, y_train, train))\n    #('std_scaler', StandardScaler())\n])\n\nX_test_tr = cat_pipeline.transform(X_test)\nX_test_tr = num_pipeline.transform(X_test_tr[num_cols])","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:44:05.685478Z","iopub.execute_input":"2022-09-07T08:44:05.685822Z","iopub.status.idle":"2022-09-07T08:44:36.846772Z","shell.execute_reply.started":"2022-09-07T08:44:05.685789Z","shell.execute_reply":"2022-09-07T08:44:36.845784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use fitted model to predict\ny_test = lr.predict(X_test_tr)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:44:54.494106Z","iopub.execute_input":"2022-09-07T08:44:54.494440Z","iopub.status.idle":"2022-09-07T08:44:54.628373Z","shell.execute_reply.started":"2022-09-07T08:44:54.494407Z","shell.execute_reply":"2022-09-07T08:44:54.627019Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:44:57.736196Z","iopub.execute_input":"2022-09-07T08:44:57.737359Z","iopub.status.idle":"2022-09-07T08:44:57.747833Z","shell.execute_reply.started":"2022-09-07T08:44:57.737262Z","shell.execute_reply":"2022-09-07T08:44:57.746586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# we can use the statsmodel library for logit regression to better understand out model. especially to see if any variables are statistically insignificant. \n# The 5th variable (site category) seems to be statistically insignificant (P > 0.05). We can take that variable out and rerun the process \nlogit_model=sm.Logit(y_train,sm.add_constant(X_train_tr))\nlogit_model\nresult=logit_model.fit()\nstats1=result.summary()\nstats2=result.summary2()\nprint(stats1)\nprint(stats2)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:45:00.945577Z","iopub.execute_input":"2022-09-07T08:45:00.946211Z","iopub.status.idle":"2022-09-07T08:45:34.953978Z","shell.execute_reply.started":"2022-09-07T08:45:00.946159Z","shell.execute_reply":"2022-09-07T08:45:34.952714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Checking for Multicollinearity. This will help us find correlated groups of variables. A high VIF (>10) signifies issues with collinearity. \n# A few variables show a high VIF meaning we need to go back and do a proper feature selection process.\n# For each X, calculate VIF and save in dataframe\nx_temp = sm.add_constant(X_train_tr)\n\nhold = ['const']\nfor i in list(X_train)[1:]:\n    hold.append(i)\n\nvif = pd.DataFrame()\nvif[\"VIF Factor\"] = [variance_inflation_factor(x_temp, i) for i in range(x_temp.shape[1])]\nvif[\"features\"] = hold\nvif.round(1)","metadata":{"execution":{"iopub.status.busy":"2022-09-07T08:45:44.320348Z","iopub.execute_input":"2022-09-07T08:45:44.320760Z","iopub.status.idle":"2022-09-07T08:48:13.624161Z","shell.execute_reply.started":"2022-09-07T08:45:44.320721Z","shell.execute_reply":"2022-09-07T08:48:13.623018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}