{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set()\nimport random","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(5566)\nnum_lines = 40428968\nsample_size = 2000000\nskip = sorted(random.sample(range(1, num_lines), num_lines-sample_size))\nskip[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df= pd.read_csv(\"../input/avazu-ctr-prediction/train.gz\", header=0, skiprows=skip)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"hour\"] = pd.to_datetime(df[\"hour\"], format=\"%y%m%d%H\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(x='click',data=df)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we can see that the number of clicks is very less as compared to the number of non clicks"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CTR current"},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"click\"].value_counts() / len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"hour\"].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"actual_hour_day\"]= df[\"hour\"].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imp= df.groupby(['actual_hour_day', 'click']).size().unstack()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"imp.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure()\nimp.plot(kind='bar', title=\"Day hours\", figsize=(16,8))\nplt.ylabel(\"count of clicks\")\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here we can see the hourly impressions/day and we can clearly see that the rate of clicks is very low as compared to non  clicks"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby('banner_pos')['click'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby('device_type')['click'].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.banner_pos.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CTR with respect to different columns"},{"metadata":{},"cell_type":"markdown","source":"for C1"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.C1.value_counts()/len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c1= df.C1.unique()\nprint(c1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ctr_list=[]\nfor i in c1:\n    ctr=df.loc[np.where((df.C1 == i))].click.value_counts() / len(df)\n    ctr_list.append(ctr)\n    print(\"C1 values {}:, CTR is {}:\".format(i,ctr))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"FOR banner_pos"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.banner_pos.value_counts() / len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"banner= df.banner_pos.unique()\nprint(banner)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ban_list=[]\nfor i in banner:\n    baa=df.loc[np.where((df.banner_pos == i))].click.value_counts() / len(df)\n    ban_list.append(baa)\n    print(\"banner values {}:, banner is {}:\".format(i,baa))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.dtypes","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"for device_type"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.device_type.value_counts() / len(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dev= df.device_type.unique()\nprint(dev)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X= df.drop(\"click\", axis=1)\n\ny= df.click","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test= train_test_split(X,y,test_size=0.3, random_state=12)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from catboost import CatBoostClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_features= [\"site_id\", \"site_domain\", \"site_category\",\n              \"app_id\", \"app_domain\", \"app_category\", \"device_id\", \n              \"device_ip\", \"device_model\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_boost= CatBoostClassifier(learning_rate=0.05, loss_function='Logloss', \n                             random_state=123, iterations=100, eval_metric=\"Logloss\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cat_boost.fit(X_train, y_train, eval_set=(X_test, y_test), cat_features= cat_features, plot=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c= cat_boost.get_feature_importance(prettified=True).sort_index(ascending=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"c.to_csv(\"feature_imp.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions= cat_boost.predict(X_test, prediction_type=\"Class\", ntree_start=0, ntree_end= cat_boost.get_best_iteration(),thread_count=-1, \n                               verbose=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}