{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Avazu CTR prediction","metadata":{"id":"gEcUzCBSvBdd"}},{"cell_type":"code","source":"# Import packages and configure notebook\nimport pandas as pd\n\npd.options.display.max_rows = 99\npd.options.display.max_columns = 99\n\n\n%matplotlib inline\n%load_ext autoreload\n%autoreload 2","metadata":{"id":"QUHI3pmqvBdh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport sklearn\nimport matplotlib.pyplot as plt\nimport imblearn\nimport seaborn as sns\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.linear_model import Ridge, LinearRegression, Lasso, RidgeCV, LassoCV, LogisticRegression\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import euclidean_distances\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.preprocessing import StandardScaler, PolynomialFeatures, RobustScaler\nfrom sklearn.pipeline import make_pipeline\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.impute import KNNImputer\nfrom sklearn.svm import LinearSVC\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score, roc_auc_score, confusion_matrix, classification_report\nfrom sklearn.decomposition import PCA\nfrom imblearn.under_sampling import RandomUnderSampler\nfrom imblearn.pipeline import make_pipeline as make_imb_pipeline\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import plot_confusion_matrix","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Column information!","metadata":{}},{"cell_type":"raw","source":"id: ad identifier\nclick: 0/1 for non-click/click\nhour: format is YYMMDDHH, so 14091123 means 23:00 on Sept. 11, 2014 UTC.\nC1 -- anonymized categorical variable\nbanner_pos\nsite_id\nsite_domain\nsite_category\napp_id\napp_domain\napp_category\ndevice_id\ndevice_ip\ndevice_model\ndevice_type\ndevice_conn_type\nC14-C21 -- anonymized categorical variables\n\n>>> There are ~40 million rows <<< This is good to know for sub-sampling","metadata":{}},{"cell_type":"markdown","source":"## Reading the data\n\nThis assumes the \"train.gz\" file is located in \"data\" folder in the folder where the notebook is.\n\nThe file is really big, so it is not recommended to load the full file in memory.\nYou can read the files in chunks and apply online learning (see also lecture notes).\nAs a start I would recommend to first develop and test your code with a smaller data sample, however don't wait too long to also test (every now and then) if it also works on the full dataset.","metadata":{"id":"Lddk1SptBX5X"}},{"cell_type":"markdown","source":"Hand in a short report (next to your code/notebook) where you discuss your observations and\nadvice for implementing the CTR prediction model:\n\n• What are your observations when exploring the dataset.\n• How did you select which variables to include in your models.\n• How you choose the parameters of your models.\n• How you split the dataset into a training and test set.\n• What the prediction accuracy is of the two models based on Log Loss metric.\n• How the calculation times of the two models compare.\n• The practical difficulties you encountered while implementing your CTR prediction models\non a real-life dataset.\n• Which model would you advice to implement?","metadata":{}},{"cell_type":"code","source":"# Reading the file in chunks\nreader = pd.read_csv(\"data/train.gz\", compression='gzip', chunksize=1000)\n\nteller=0\n\nfor df_chunk in reader:\n    teller+=1\n    # Apply logic on current chunk of data contained in the df_chunk DataFrame\n    print(len(df_chunk),\" - \",teller)\n    \n    if teller == 100:\n        break","metadata":{"id":"TMlysQu9vBdi","scrolled":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# I called this dataframe \"df\" so it is also easier to use later on when we use bigger samples/sets.\n\npath = '\\\"data/train.gz\\\"\"'\npath = \n# Read only a sample of the data (first nrows)\ndf = pd.read_csv(\"data/train.gz\", compression='gzip', nrows=100000)\n\ndf\n\n# This sample is very not random (just the first 100k rows), we need to that better maybe.","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# skiprows idea from stack because we need some sort of sample, not just the top rows to start of with\n# https://stackoverflow.com/questions/22258491/read-a-small-random-sample-from-a-big-csv-file-into-a-python-data-frame\n\nimport random\np = 0.001  # 0.1% of the lines\n\ndf = pd.read_csv(\"data/train.gz\", compression='gzip',\n         header=0, \n         skiprows=lambda i: i>0 and random.random() > p\n)\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Clean up the data and split it","metadata":{}},{"cell_type":"code","source":"# Change the [\"hour\"] column into something useful. \n# We only need hour and day of week since year/month/day is irrelevant for an 11 day dataset.\n\n# The day of the week with Monday=0, Sunday=6.\n# https://pandas.pydata.org/docs/reference/api/pandas.Series.dt.dayofweek.html#pandas.Series.dt.dayofweek\n\ndf[\"n_hour\"] = pd.to_datetime(df[\"hour\"], format=\"%y%m%d%H\").dt.hour\ndf[\"n_weekday\"] = pd.to_datetime(df[\"hour\"], format=\"%y%m%d%H\").dt.dayofweek\n\n# Drop the original timestamp, no exact time will happen again (historical data) so best to remove.\ndf = df.drop([\"hour\"], axis=1)\ndf","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Drop device_ip the data is masked and subsampled, potential value unclear.\ndf = df.drop([\"device_ip\"], axis=1)\n\n# Drop the id column\ndf = df.drop([\"id\"],axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Put the target variable in a seperate df\ny = df[\"click\"]\n\n# Create new df with variables except for the target one\nX = df.drop([\"click\"], axis=1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split it up in train and test sets. Train/test 0.8/0.2 because that's what we've been using so far doing ML.\nX_train, X_test, y_train, y_test = train_test_split(\n    X, y, random_state=22, train_size=0.8, test_size =0.2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# sense check on distributions, seems OK. Not checking the y_test for academic purity ;-)\nprint(y_train.value_counts(normalize=True))\nprint(y.value_counts(normalize=True))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 1 - LogReg","metadata":{}},{"cell_type":"code","source":"# All the pre-prep stuff for re-use with other algorithms as well.\n\n# Save the original column names\nX_columns = X.columns\n\n# Change all the categorical features into numbers -- probably also needed for the other model?\nfrom sklearn.preprocessing import OrdinalEncoder\nenc = OrdinalEncoder()\nX = enc.fit_transform(X)\nX = pd.DataFrame(X)\n\n# Give back original column names\nX.columns = X_columns\n\n# Split it up in train and test sets. Train/test 0.8/0.2 because that's what we've been using so far doing ML.\nX_train, X_test, y_train, y_test = train_test_split(\n    X, y, random_state=22, train_size=0.8, test_size =0.2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Load the algo\nfrom sklearn.linear_model import LogisticRegression\n\nlogreg = LogisticRegression(multi_class=\"multinomial\", solver=\"lbfgs\")\n\nparam_grid = {\"C\":np.logspace(-3,3,7)}\n\ngrid = GridSearchCV(logreg, param_grid = param_grid, return_train_score = True, cv=5, n_jobs=2)\ngrid.fit(X_train,y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize scoring and other stuff\n\nprint(\"LOGREG train on train: \", grid.score(X_train,y_train), \"test on test: \", grid.score(X_test,y_test))\n\n# Visualize the impact of tweaking C, although the highest score on test was achieved with the default 1-setting.\n\nresults_grid = pd.DataFrame(grid.cv_results_)\n\nresults_grid.plot('param_C', 'mean_train_score', label=\"LogReg Mean Train Score\", ax=plt.gca())\nresults_grid.plot('param_C', 'mean_test_score', label=\"LogReg Mean Test Score\", ax=plt.gca())\nplt.legend()\nplt.xscale(\"log\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print standard classification report\nprint(classification_report(y_test, grid.predict(X_test)))\n\n# Print feature importance bar chart\ntiteltje = \"Feature Importance for target value \", grid.best_estimator_.classes_[0]\n\n# Label the labels\ncoef = pd.Series(grid.best_estimator_.coef_[0], index = X.columns)\nimp_coef = coef.sort_values()\n\n# Plot the plot\nimport matplotlib\nmatplotlib.rcParams['figure.figsize'] = (8.0, 10.0)\nimp_coef.plot(kind = \"barh\")\nplt.title(titeltje)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# log_loss metric\n\nfrom sklearn.metrics import log_loss\nlog_loss(y_test, grid.predict(X_test))","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# SGD Classifier","metadata":{}},{"cell_type":"code","source":"# All the pre-prep stuff for re-use with other algorithms as well.\n\n# Save the original column names\nX_columns = X.columns\n\n# Change all the categorical features into numbers -- probably also needed for the other model?\nfrom sklearn.preprocessing import OrdinalEncoder\nenc = OrdinalEncoder()\nX = enc.fit_transform(X)\nX = pd.DataFrame(X)\n\n# Give back original column names\nX.columns = X_columns\n\n# Split it up in train and test sets. Train/test 0.8/0.2 because that's what we've been using so far doing ML.\nX_train, X_test, y_train, y_test = train_test_split(\n    X, y, random_state=22, train_size=0.8, test_size =0.2)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.linear_model import SGDClassifier\n\nsgd = SGDClassifier()\n#sgd.fit(X_train, y_train)\n\nparam_grid_sgd = {}\ngrid = GridSearchCV(sgd, param_grid = param_grid_sgd  ,return_train_score = True, cv=5, n_jobs=2)\n\ngrid.fit(X_train, y_train)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print standard classification report\nprint(classification_report(y_test, grid.predict(X_test)))\n\n# Print feature importance bar chart\ntiteltje = \"Feature Importance for target value \", grid.best_estimator_.classes_[0]\n\n# Label the labels\ncoef = pd.Series(grid.best_estimator_.coef_[0], index = X.columns)\nimp_coef = coef.sort_values()\n\n# Plot the plot\nimport matplotlib\nmatplotlib.rcParams['figure.figsize'] = (8.0, 10.0)\nimp_coef.plot(kind = \"barh\")\nplt.title(titeltje)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# log_loss metric\n\nfrom sklearn.metrics import log_loss\nlog_loss(y_test, grid.predict(X_test))","metadata":{},"execution_count":null,"outputs":[]}]}