{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 模型引用自 Oluyemi Jegede\n\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-22T04:16:34.604893Z","iopub.execute_input":"2022-09-22T04:16:34.605469Z","iopub.status.idle":"2022-09-22T04:16:34.642598Z","shell.execute_reply.started":"2022-09-22T04:16:34.605384Z","shell.execute_reply":"2022-09-22T04:16:34.641551Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import necessary libraries\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom category_encoders import TargetEncoder\nimport random\nimport gzip\nimport seaborn as sns # 引入Seaborn","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:16:34.644432Z","iopub.execute_input":"2022-09-22T04:16:34.644714Z","iopub.status.idle":"2022-09-22T04:16:36.471213Z","shell.execute_reply.started":"2022-09-22T04:16:34.644686Z","shell.execute_reply":"2022-09-22T04:16:36.470475Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data paths\ntrain_data_path = \"/kaggle/input/avazu-ctr-prediction/train.gz\"\ntest_data_path = \"/kaggle/input/avazu-ctr-prediction/test.gz\"\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:16:36.472397Z","iopub.execute_input":"2022-09-22T04:16:36.473596Z","iopub.status.idle":"2022-09-22T04:16:36.478477Z","shell.execute_reply.started":"2022-09-22T04:16:36.473560Z","shell.execute_reply":"2022-09-22T04:16:36.477389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# pd.set_option('display.float_format', lambda x: '%.0f' % x)\n# get the test data\ntest_data = pd.read_csv(test_data_path, chunksize = 1000000)\ntest_df = pd.DataFrame()\n\nfor chunk in test_data:\n    test_df = pd.concat([test_df, chunk], axis = 0)\n# print(test_df['id'])\n\n# # get the sampleSubmission data\n# sampleSubmission_data = pd.read_csv(sampleSubmission_data_path, chunksize = 1000000)\n# sampleSubmission_df = pd.DataFrame()\n\n# for chunk in sampleSubmission_data:\n#     sampleSubmission_df = pd.concat([sampleSubmission_df, chunk], axis = 0)\n# print(sampleSubmission_df['id'])","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:16:36.479533Z","iopub.execute_input":"2022-09-22T04:16:36.480107Z","iopub.status.idle":"2022-09-22T04:16:51.004869Z","shell.execute_reply.started":"2022-09-22T04:16:36.480083Z","shell.execute_reply":"2022-09-22T04:16:51.003433Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"ID 以科學記號呈現 的問題","metadata":{}},{"cell_type":"code","source":"# # get train data sample using pd chunks. Each chunk has a size of 1000000\n# train_data = pd.read_csv(train_data_path, chunksize = 1000000)\n# train_df = pd.DataFrame()\n\n# for chunk in train_data:\n#     train_df = pd.concat([train_df, chunk.sample(frac=.1, replace=False, random_state=123)], axis=0)\n\nn = 40428967 #数据集中的记录总数\nsample_size = 4577464\nskip_values = sorted(random.sample(range(1,n), n-sample_size)) \nparse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')\n \nwith gzip.open(train_data_path) as f:\n  train_df = pd.read_csv(f, parse_dates = ['hour'], date_parser = parse_date, skiprows = skip_values)\n\n\n# # get train data sample using pd chunks. Each chunk has a size of 1000000\n# train_data = pd.read_csv(train_data_path, chunksize = 1000000)\n# train_df = pd.DataFrame()\n\n# for chunk in train_data:\n#     train_df = pd.concat([train_df, chunk.sample(frac=.1, replace=False, random_state=123)], axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:16:51.008082Z","iopub.execute_input":"2022-09-22T04:16:51.008474Z","iopub.status.idle":"2022-09-22T04:20:13.217502Z","shell.execute_reply.started":"2022-09-22T04:16:51.008430Z","shell.execute_reply":"2022-09-22T04:20:13.216461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"從train data 中 隨機取樣 4577464 ","metadata":{}},{"cell_type":"code","source":"# some feature engineering and pipeline building\n# get the day of the weeko through a pipeline class\nclass DateTransformer(BaseEstimator, TransformerMixin):\n    '''Custom transformer for making date variables'''\n    def __init__(self, data_col):\n        self.data_col = data_col\n        \n    def build_date(self, hour):\n        day = str(hour)[4:6]\n        month = str(hour)[2:4]\n        year = '20'+str(hour)[0:2]\n        return day + '-' + month + '-' + year\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, X, y = None):\n        X_ = X.copy()\n        X_['fdate'] = pd.to_datetime(X_.hour.apply(self.build_date))\n        X_['weekday'] = X_['fdate'].dt.dayofweek.astype(str)\n        return X_\n\n# many of our categorical variables have high cardinality. let's try target encoding/mean encoding on them\nclass EncoderTransformer(BaseEstimator, TransformerMixin):\n    '''Custom transformer for encoding categorical variables'''\n    def __init__(self, encoder, target, train):\n        self.encoder = encoder\n        self.target = target\n        self.train = train\n        \n    def fit(self, X, y=None):\n        return self\n    \n    def transform(self, X, y=None):\n        X_ = X.copy()\n        if self.train == 1:\n            X_ = self.encoder.fit_transform(X_, self.target)\n        elif self.train == 0:\n            X_ = self.encoder.transform(X_, self.target)\n        return X_","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:13.218587Z","iopub.execute_input":"2022-09-22T04:20:13.218832Z","iopub.status.idle":"2022-09-22T04:20:13.230065Z","shell.execute_reply.started":"2022-09-22T04:20:13.218810Z","shell.execute_reply":"2022-09-22T04:20:13.228999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"資料格式處理","metadata":{}},{"cell_type":"code","source":"cat_cols = ['hour','C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n       'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n       'device_model', 'device_type', 'device_conn_type', 'C14', 'C15', 'C16',\n       'C17', 'C18', 'C19', 'C20', 'C21']","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:13.231417Z","iopub.execute_input":"2022-09-22T04:20:13.231661Z","iopub.status.idle":"2022-09-22T04:20:13.244480Z","shell.execute_reply.started":"2022-09-22T04:20:13.231639Z","shell.execute_reply":"2022-09-22T04:20:13.243533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# declare target encoder\nencoder = TargetEncoder(cols = ['C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n       'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n       'device_model', 'device_type', 'device_conn_type', 'C14', 'C15', 'C16',\n       'C17', 'C18', 'C19', 'C20', 'C21'])","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:13.246471Z","iopub.execute_input":"2022-09-22T04:20:13.247019Z","iopub.status.idle":"2022-09-22T04:20:13.255767Z","shell.execute_reply.started":"2022-09-22T04:20:13.246987Z","shell.execute_reply":"2022-09-22T04:20:13.254567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# make training and test sets\nX_train = train_df.drop(['click', 'id'], axis=1)\ny_train = train_df.click.values\nX_test = test_df.drop('id', axis=1)\n# test_df.to_csv(\"./test_df.csv\",index=False, sep=\",\", mode='w', header=True)#point_table","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:13.257746Z","iopub.execute_input":"2022-09-22T04:20:13.258059Z","iopub.status.idle":"2022-09-22T04:20:14.270574Z","shell.execute_reply.started":"2022-09-22T04:20:13.258032Z","shell.execute_reply":"2022-09-22T04:20:14.269289Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_cols = ['C1', 'banner_pos', 'site_id', 'site_domain', 'site_category',\n       'app_id', 'app_domain', 'app_category', 'device_id', 'device_ip',\n       'device_model', 'device_type', 'device_conn_type', 'C14', 'C15', 'C16',\n       'C17', 'C18', 'C19', 'C20', 'C21']","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:14.271846Z","iopub.execute_input":"2022-09-22T04:20:14.272168Z","iopub.status.idle":"2022-09-22T04:20:14.277570Z","shell.execute_reply.started":"2022-09-22T04:20:14.272141Z","shell.execute_reply":"2022-09-22T04:20:14.276471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create pipeline steps\ntrain = 1\ncat_pipeline = Pipeline(steps=[\n    ('date_maker', DateTransformer('hour')),\n    ('en_coder', EncoderTransformer(encoder, y_train, train))\n    #('std_scaler', StandardScaler())\n])","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:14.278725Z","iopub.execute_input":"2022-09-22T04:20:14.278990Z","iopub.status.idle":"2022-09-22T04:20:14.288702Z","shell.execute_reply.started":"2022-09-22T04:20:14.278968Z","shell.execute_reply":"2022-09-22T04:20:14.287814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# create second pipeline step. alternatively we could have combined the two pipelines using the Column Transformer module\nnum_pipeline = Pipeline(steps=[\n    ('std_scaler', StandardScaler())\n])","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:14.289855Z","iopub.execute_input":"2022-09-22T04:20:14.290080Z","iopub.status.idle":"2022-09-22T04:20:14.298855Z","shell.execute_reply.started":"2022-09-22T04:20:14.290059Z","shell.execute_reply":"2022-09-22T04:20:14.298042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fit and transform our trainingv data on our pieplines\nX_train_tr = cat_pipeline.fit_transform(X_train)\nX_train_tr = num_pipeline.fit_transform(X_train_tr[num_cols])","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:20:14.300100Z","iopub.execute_input":"2022-09-22T04:20:14.300417Z","iopub.status.idle":"2022-09-22T04:24:07.217537Z","shell.execute_reply.started":"2022-09-22T04:20:14.300391Z","shell.execute_reply":"2022-09-22T04:24:07.215606Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # we want to use logistic regression because of the binary nature of our target. \n# # but we have some class imbalance so let's first use a gird search to find the right weights for our model\n# lr = LogisticRegression(solver='newton-cg')\n\n# #Setting the range for class weights\n# weights = np.linspace(0.0,0.99,6)\n\n\n# #Creating a dictionary grid for grid search\n# param_grid = {'class_weight': [{0:x, 1:1.0-x} for x in weights]}\n\n# #Fitting grid search to the train data with 5 folds\n# gridsearch = GridSearchCV(estimator= lr, \n#                           param_grid= param_grid,\n#                           cv=StratifiedKFold(), \n#                           n_jobs=-1, \n#                           scoring='f1', \n#                           verbose=2).fit(X_train_tr, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:24:07.222734Z","iopub.execute_input":"2022-09-22T04:24:07.223128Z","iopub.status.idle":"2022-09-22T04:24:07.230374Z","shell.execute_reply.started":"2022-09-22T04:24:07.223090Z","shell.execute_reply":"2022-09-22T04:24:07.229001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # let's check out how the f1 score changes with different weight ratios\n# sns.set_style('whitegrid')\n# plt.figure(figsize=(12,8))\n# weigh_data = pd.DataFrame({ 'score': gridsearch.cv_results_['mean_test_score'], 'weight': (1- weights)})\n# sns.lineplot(weigh_data['weight'], weigh_data['score']);\n# plt.xlabel('Weight for class 0')\n# plt.ylabel('F1 score')\n# plt.xticks([round(i/10,1) for i in range(0,11,1)])\n# plt.title('Scoring for different class weights', fontsize=24)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:24:07.231772Z","iopub.execute_input":"2022-09-22T04:24:07.232761Z","iopub.status.idle":"2022-09-22T04:24:07.242689Z","shell.execute_reply.started":"2022-09-22T04:24:07.232720Z","shell.execute_reply":"2022-09-22T04:24:07.241686Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# declare sklearn logistic regressor and fit the model using the optimal classw weights from above\nlr = LogisticRegression(class_weight={0: 0.8, 1: 0.2}, max_iter=5000)\nlr.fit(X_train_tr, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:24:07.244037Z","iopub.execute_input":"2022-09-22T04:24:07.244354Z","iopub.status.idle":"2022-09-22T04:24:27.515080Z","shell.execute_reply.started":"2022-09-22T04:24:07.244328Z","shell.execute_reply":"2022-09-22T04:24:27.514299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# # transform the test set using the same pipeline\ntrain = 0\ncat_pipeline = Pipeline(steps=[\n    ('date_maker', DateTransformer('hour')),\n    ('en_coder', EncoderTransformer(encoder, y_train, train))\n    #('std_scaler', StandardScaler())\n])\n\nX_test_tr = cat_pipeline.transform(X_test)\nX_test_tr = num_pipeline.transform(X_test_tr[num_cols])\n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:24:27.516162Z","iopub.execute_input":"2022-09-22T04:24:27.516627Z","iopub.status.idle":"2022-09-22T04:25:25.678054Z","shell.execute_reply.started":"2022-09-22T04:24:27.516597Z","shell.execute_reply":"2022-09-22T04:25:25.677059Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# use fitted model to predict\ny_test = lr.predict(X_test_tr)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:25:25.679366Z","iopub.execute_input":"2022-09-22T04:25:25.679715Z","iopub.status.idle":"2022-09-22T04:25:25.772736Z","shell.execute_reply.started":"2022-09-22T04:25:25.679683Z","shell.execute_reply":"2022-09-22T04:25:25.771667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test\npd.DataFrame(y_test).to_csv(\"./y_test.csv\",index=False, sep=\",\", mode='w', header=True)#point_table\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:25:25.774006Z","iopub.execute_input":"2022-09-22T04:25:25.774294Z","iopub.status.idle":"2022-09-22T04:25:28.798022Z","shell.execute_reply.started":"2022-09-22T04:25:25.774267Z","shell.execute_reply":"2022-09-22T04:25:28.796830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"用模型預測test data的 y","metadata":{}},{"cell_type":"code","source":"# test_data_path = \"/kaggle/input/avazu-ctr-prediction/sampleSubmission.gz\"\n# # get the test data\n# pd.set_option('display.float_format', lambda x: '%.0f' % x)\n# test_data = pd.read_csv(test_data_path, chunksize = 1000000)\n# test_df = pd.DataFrame()\n\n# for chunk in test_data:\n#     test_df = pd.concat([test_df, chunk], axis = 0)\n\n\n# get the sampleSubmission data\nsampleSubmission_data_path = \"/kaggle/input/avazu-ctr-prediction/sampleSubmission.gz\"\nsampleSubmission_data = pd.read_csv(sampleSubmission_data_path, chunksize = 1000000)\nsampleSubmission_df = pd.DataFrame()\n\nfor chunk in sampleSubmission_data:\n    sampleSubmission_df = pd.concat([sampleSubmission_df, chunk], axis = 0)\n\nprint(sampleSubmission_df)\n    \nsampleSubmission_df= sampleSubmission_df[['id']]\nsampleSubmission_df['click'] = y_test\n\nresult_df= sampleSubmission_df[['id','click']]\nprint(result_df)\npd.DataFrame(result_df).to_csv(\"./submission.csv\",float_format='%.0f',index=False, sep=\",\", mode='w', header=True)#point_table\n\n\n","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:25:28.799274Z","iopub.execute_input":"2022-09-22T04:25:28.799817Z","iopub.status.idle":"2022-09-22T04:25:36.312091Z","shell.execute_reply.started":"2022-09-22T04:25:28.799769Z","shell.execute_reply":"2022-09-22T04:25:36.310576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"merge id欄位 和 test y 預測值 欄位，以輸出最終結果","metadata":{}},{"cell_type":"code","source":"# we can use the statsmodel library for logit regression to better understand out model. especially to see if any variables are statistically insignificant. \n# The 5th variable (site category) seems to be statistically insignificant (P > 0.05). We can take that variable out and rerun the process \nlogit_model=sm.Logit(y_train,sm.add_constant(X_train_tr))\nlogit_model\nresult=logit_model.fit()\nstats1=result.summary()\nstats2=result.summary2()\nprint(stats1)\nprint(stats2)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:25:36.313919Z","iopub.execute_input":"2022-09-22T04:25:36.314257Z","iopub.status.idle":"2022-09-22T04:25:55.126013Z","shell.execute_reply.started":"2022-09-22T04:25:36.314228Z","shell.execute_reply":"2022-09-22T04:25:55.124500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"模型說明","metadata":{}},{"cell_type":"code","source":"# Checking for Multicollinearity. This will help us find correlated groups of variables. A high VIF (>10) signifies issues with collinearity. \n# A few variables show a high VIF meaning we need to go back and do a proper feature selection process.\n# For each X, calculate VIF and save in dataframe\nx_temp = sm.add_constant(X_train_tr)\n\nhold = ['const']\nfor i in list(X_train)[1:]:\n    hold.append(i)\n\nvif = pd.DataFrame()\nvif[\"VIF Factor\"] = [variance_inflation_factor(x_temp, i) for i in range(x_temp.shape[1])]\nvif[\"features\"] = hold\nvif.round(1)","metadata":{"execution":{"iopub.status.busy":"2022-09-22T04:25:55.127487Z","iopub.execute_input":"2022-09-22T04:25:55.127897Z","iopub.status.idle":"2022-09-22T04:27:23.565223Z","shell.execute_reply.started":"2022-09-22T04:25:55.127862Z","shell.execute_reply":"2022-09-22T04:27:23.564310Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"特徵重要性","metadata":{}}]}