{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-09-21T03:29:37.637555Z","iopub.execute_input":"2022-09-21T03:29:37.638510Z","iopub.status.idle":"2022-09-21T03:29:37.647528Z","shell.execute_reply.started":"2022-09-21T03:29:37.638469Z","shell.execute_reply":"2022-09-21T03:29:37.646331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import necessary libraries\nimport pandas as pd\nimport numpy as np\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.compose import ColumnTransformer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nimport statsmodels.api as sm\nfrom statsmodels.stats.outliers_influence import variance_inflation_factor\nfrom category_encoders import TargetEncoder\nimport random\nimport gzip\nimport seaborn as sns # 引入Seaborn","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:29:37.650324Z","iopub.execute_input":"2022-09-21T03:29:37.650755Z","iopub.status.idle":"2022-09-21T03:29:37.660255Z","shell.execute_reply.started":"2022-09-21T03:29:37.650709Z","shell.execute_reply":"2022-09-21T03:29:37.658618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# data paths\ntrain_data_path = \"/kaggle/input/avazu-ctr-prediction/train.gz\"\ntest_data_path = \"/kaggle/input/avazu-ctr-prediction/test.gz\"","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:29:37.662210Z","iopub.execute_input":"2022-09-21T03:29:37.663055Z","iopub.status.idle":"2022-09-21T03:29:37.673175Z","shell.execute_reply.started":"2022-09-21T03:29:37.663014Z","shell.execute_reply":"2022-09-21T03:29:37.671926Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n = 40428967 #数据集中的记录总数\nsample_size = 1000000\nskip_values = sorted(random.sample(range(1,n), n-sample_size)) \nparse_date = lambda val : pd.datetime.strptime(val, '%y%m%d%H')\n \nwith gzip.open(train_data_path) as f:\n  data_df = pd.read_csv(f, parse_dates = ['hour'], date_parser = parse_date, skiprows = skip_values)\ntrain_df=data_df.copy()\n\n# get the test data\ntest_data = pd.read_csv(test_data_path, chunksize = 1000000)\ntest_df = pd.DataFrame()\n\nfor chunk in test_data:\n    test_df = pd.concat([test_df, chunk], axis = 0)\ntest_df = test_df.loc[0:4042896]\n\ndata_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:29:37.678800Z","iopub.execute_input":"2022-09-21T03:29:37.679297Z","iopub.status.idle":"2022-09-21T03:33:15.365134Z","shell.execute_reply.started":"2022-09-21T03:29:37.679261Z","shell.execute_reply":"2022-09-21T03:33:15.364015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#提取标签中0和1的数量，并存入df_click中\ndf_click = data_df['click'].value_counts()\n#打印结果\nprint(df_click)\n#计算1、0的占比\npos_proportion = df_click[1]/(df_click[1] + df_click[0])\nneg_proportion = df_click[0]/(df_click[1] + df_click[0])\n#打印结果\nprint('positive sample count：%.2f'%(pos_proportion))\nprint('negtive sample count：%.2f'%(neg_proportion))\n\n#可视化结果\nplt.figure(figsize=(10,6))\ndf_click.plot(kind = 'bar', width = 0.2, rot = 0)\nplt.title('positive sample distribution',fontsize=16)\n# 添加y轴标签\nplt.ylabel('sample count',fontsize=14)\n# 为每个条形图添加数值标签\nfor x,y in enumerate(df_click.values):\n    plt.text(x,y+1000,'%s'%y,ha='center',fontsize=13)\nplt.savefig('./pic_1')\n","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:33:15.366763Z","iopub.execute_input":"2022-09-21T03:33:15.367123Z","iopub.status.idle":"2022-09-21T03:33:15.688689Z","shell.execute_reply.started":"2022-09-21T03:33:15.367093Z","shell.execute_reply":"2022-09-21T03:33:15.687716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\n# data_df['hour'] = pd.to_datetime(data_df['hour'],format='%y%m%d%H')\ndata_df[\"hour_str\"] = data_df['hour'].astype(str)\ndata_df[\"hr\"] = data_df[\"hour_str\"].str[11:13]\ndata_df['Datacollecttime_pd_format'] = pd.to_datetime(data_df['hour_str'], format='%Y-%m-%d %H')\ndata_df[\"weekday\"] = data_df[\"Datacollecttime_pd_format\"].dt.dayofweek+1 #1: Mon, 7: Sun\nprint(data_df[['hour_str','weekday','hr']])","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:33:15.690222Z","iopub.execute_input":"2022-09-21T03:33:15.690839Z","iopub.status.idle":"2022-09-21T03:33:18.249834Z","shell.execute_reply.started":"2022-09-21T03:33:15.690793Z","shell.execute_reply":"2022-09-21T03:33:18.248344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_df = data_df.groupby(by=[\"weekday\",\"hr\"])[[\"click\"]].agg(['sum','count']).reset_index()\ndata_df.columns = [\"_\".join(x) for x in data_df.columns.ravel()]\ndata_df['click_rate']=data_df.click_sum/data_df.click_count\nprint(data_df)\ng = sns.FacetGrid(data_df, col=\"weekday_\", col_wrap=7, height=4)\ng.map(sns.lineplot, \"hr_\", \"click_rate\")","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:33:18.253023Z","iopub.execute_input":"2022-09-21T03:33:18.253408Z","iopub.status.idle":"2022-09-21T03:33:21.373246Z","shell.execute_reply.started":"2022-09-21T03:33:18.253372Z","shell.execute_reply":"2022-09-21T03:33:21.371914Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g2 = sns.FacetGrid(data_df, col=\"weekday_\", col_wrap=7, height=4)\ng2.map(sns.lineplot, \"hr_\", \"click_sum\")","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:33:21.375286Z","iopub.execute_input":"2022-09-21T03:33:21.375880Z","iopub.status.idle":"2022-09-21T03:33:24.229708Z","shell.execute_reply.started":"2022-09-21T03:33:21.375842Z","shell.execute_reply":"2022-09-21T03:33:24.228474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"g3= sns.lineplot(data=data_df, x=\"hr_\", y=\"click_rate\")","metadata":{"execution":{"iopub.status.busy":"2022-09-21T03:33:24.231480Z","iopub.execute_input":"2022-09-21T03:33:24.231990Z","iopub.status.idle":"2022-09-21T03:33:25.132356Z","shell.execute_reply.started":"2022-09-21T03:33:24.231920Z","shell.execute_reply":"2022-09-21T03:33:25.131250Z"},"trusted":true},"execution_count":null,"outputs":[]}]}