{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"![\"Kaggle\"](\"https://kaggle.com/static/images/open-in-kaggle.svg\" "\\"Open")
","metadata":{}},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"import os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))","metadata":{"execution":{"iopub.status.busy":"2024-05-30T01:10:01.451454Z","iopub.execute_input":"2024-05-30T01:10:01.451892Z","iopub.status.idle":"2024-05-30T01:10:01.497807Z","shell.execute_reply.started":"2024-05-30T01:10:01.451857Z","shell.execute_reply":"2024-05-30T01:10:01.496455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Pyspark","metadata":{}},{"cell_type":"code","source":"!pip install -q pyspark","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:28:03.327264Z","iopub.execute_input":"2024-05-30T02:28:03.327745Z","iopub.status.idle":"2024-05-30T02:28:53.732684Z","shell.execute_reply.started":"2024-05-30T02:28:03.32771Z","shell.execute_reply":"2024-05-30T02:28:53.729953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.sql import SparkSession\nspark = SparkSession.builder.master('local[*]').appName('Avazu CTR').getOrCreate()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:28:53.737892Z","iopub.execute_input":"2024-05-30T02:28:53.739393Z","iopub.status.idle":"2024-05-30T02:29:00.619087Z","shell.execute_reply.started":"2024-05-30T02:28:53.739344Z","shell.execute_reply":"2024-05-30T02:29:00.617843Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import Library","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom pyspark.ml.feature import StringIndexer\nfrom pyspark.ml.stat import Correlation\nfrom pyspark.ml.feature import VectorAssembler\nfrom pyspark.ml.feature import RobustScaler\nfrom pyspark.ml.classification import DecisionTreeClassifier\nfrom pyspark.ml.evaluation import MulticlassClassificationEvaluator","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:29:00.621452Z","iopub.execute_input":"2024-05-30T02:29:00.622475Z","iopub.status.idle":"2024-05-30T02:29:02.669855Z","shell.execute_reply.started":"2024-05-30T02:29:00.622416Z","shell.execute_reply":"2024-05-30T02:29:02.66895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load the Training Data","metadata":{}},{"cell_type":"markdown","source":"total rows : 40428967","metadata":{}},{"cell_type":"code","source":"total_data = 40428967\n\n# load 10% data\ntrain_df = spark.read.csv('/kaggle/input/avazu-ctr-prediction/train.gz', header=True, inferSchema=True).limit(int(total_data * 0.1))","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:29:02.672109Z","iopub.execute_input":"2024-05-30T02:29:02.672658Z","iopub.status.idle":"2024-05-30T02:32:52.578013Z","shell.execute_reply.started":"2024-05-30T02:29:02.672625Z","shell.execute_reply":"2024-05-30T02:32:52.575374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.printSchema()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:52.580101Z","iopub.execute_input":"2024-05-30T02:32:52.580608Z","iopub.status.idle":"2024-05-30T02:32:52.594755Z","shell.execute_reply.started":"2024-05-30T02:32:52.58057Z","shell.execute_reply":"2024-05-30T02:32:52.593651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show how many data\ntrain_df.count()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:52.596684Z","iopub.execute_input":"2024-05-30T02:32:52.597775Z","iopub.status.idle":"2024-05-30T02:32:58.684691Z","shell.execute_reply.started":"2024-05-30T02:32:52.597728Z","shell.execute_reply":"2024-05-30T02:32:58.683476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.sql.types import StringType, IntegerType\nfrom pyspark.sql.functions import to_timestamp, col, hour","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:58.686549Z","iopub.execute_input":"2024-05-30T02:32:58.687711Z","iopub.status.idle":"2024-05-30T02:32:58.693346Z","shell.execute_reply.started":"2024-05-30T02:32:58.687665Z","shell.execute_reply":"2024-05-30T02:32:58.692136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.select('hour').show(3)","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:58.695397Z","iopub.execute_input":"2024-05-30T02:32:58.696458Z","iopub.status.idle":"2024-05-30T02:32:59.038371Z","shell.execute_reply.started":"2024-05-30T02:32:58.696415Z","shell.execute_reply":"2024-05-30T02:32:59.037134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = train_df.withColumn('datetime', to_timestamp(col('hour').cast(StringType()), format=\"yyMMddHH\").cast(\"timestamp\"))\ntrain_df = train_df.withColumn('hour', hour(col('datetime')))","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:59.039816Z","iopub.execute_input":"2024-05-30T02:32:59.040333Z","iopub.status.idle":"2024-05-30T02:32:59.187989Z","shell.execute_reply.started":"2024-05-30T02:32:59.040291Z","shell.execute_reply":"2024-05-30T02:32:59.186741Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.select('datetime', 'hour').show(3)","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:59.194228Z","iopub.execute_input":"2024-05-30T02:32:59.195174Z","iopub.status.idle":"2024-05-30T02:32:59.549325Z","shell.execute_reply.started":"2024-05-30T02:32:59.195123Z","shell.execute_reply":"2024-05-30T02:32:59.54816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.printSchema()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:32:59.550587Z","iopub.execute_input":"2024-05-30T02:32:59.551013Z","iopub.status.idle":"2024-05-30T02:32:59.55863Z","shell.execute_reply.started":"2024-05-30T02:32:59.550976Z","shell.execute_reply":"2024-05-30T02:32:59.557574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"# eda\ntrain_df.describe().toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:29:46.733841Z","iopub.execute_input":"2024-05-29T12:29:46.734876Z","iopub.status.idle":"2024-05-29T12:33:38.856267Z","shell.execute_reply.started":"2024-05-29T12:29:46.734835Z","shell.execute_reply":"2024-05-29T12:33:38.854535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.select('click', 'hour', 'C1', 'banner_pos', 'device_type', 'device_conn_type', 'C14').describe().toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T01:28:59.986876Z","iopub.execute_input":"2024-05-30T01:28:59.987481Z","iopub.status.idle":"2024-05-30T01:29:23.811712Z","shell.execute_reply.started":"2024-05-30T01:28:59.987435Z","shell.execute_reply":"2024-05-30T01:29:23.810816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.select('C15', 'C16', 'C17', 'C18', 'C19', 'C20', 'C21').describe().toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T01:30:33.062471Z","iopub.execute_input":"2024-05-30T01:30:33.062992Z","iopub.status.idle":"2024-05-30T01:30:51.217838Z","shell.execute_reply.started":"2024-05-30T01:30:33.062952Z","shell.execute_reply":"2024-05-30T01:30:51.216626Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# number of click\nprint(f'Number of click: {train_df[train_df[\"click\"] == 1].count()}')\n\n# number of non-click\nprint(f'Number of non-click: {train_df[train_df[\"click\"] == 0].count()}')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:33:38.85956Z","iopub.execute_input":"2024-05-29T12:33:38.860268Z","iopub.status.idle":"2024-05-29T12:34:09.009462Z","shell.execute_reply.started":"2024-05-29T12:33:38.860223Z","shell.execute_reply":"2024-05-29T12:34:09.008121Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import warnings\nwarnings.filterwarnings('ignore')","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:38:09.504557Z","iopub.execute_input":"2024-05-30T02:38:09.505419Z","iopub.status.idle":"2024-05-30T02:38:09.510113Z","shell.execute_reply.started":"2024-05-30T02:38:09.50538Z","shell.execute_reply":"2024-05-30T02:38:09.509045Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n train_df.select('hour', 'click')\n .crosstab('hour', 'click')\n .withColumn('hour_click', col('hour_click').cast(IntegerType()))\n .sort('hour_click')\n .withColumn('click_rate', col('1') / (col('0') + col('1')))\n .show() \n)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T15:44:22.308268Z","iopub.execute_input":"2024-05-29T15:44:22.308766Z","iopub.status.idle":"2024-05-29T15:45:03.068171Z","shell.execute_reply.started":"2024-05-29T15:44:22.308732Z","shell.execute_reply":"2024-05-29T15:45:03.067062Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.histplot(x='hour', hue='click', data=train_df.select('hour', 'click').toPandas(), multiple='stack', discrete=True)\nax.set_xticks(list(range(24)), list(range(24)))\n\nax2 = ax.twinx()\nclick_rate = (\n train_df.select('hour', 'click')\n .crosstab('hour', 'click')\n .withColumn('hour_click', col('hour_click').cast(IntegerType()))\n .sort('hour_click')\n .withColumn('click_rate', col('1') / (col('0') + col('1')))\n)\nsns.lineplot(x='hour_click', y='click_rate',data=click_rate.toPandas(),marker='s',color='red',ax=ax2)\nax2.set_yticks([x/100 for x in range(10, 22, 2)], [x/100 for x in range(10, 22, 2)])\n\nplt.suptitle('Click Count and Rate by Hour')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T15:58:12.030228Z","iopub.execute_input":"2024-05-29T15:58:12.030902Z","iopub.status.idle":"2024-05-29T15:59:53.29154Z","shell.execute_reply.started":"2024-05-29T15:58:12.030844Z","shell.execute_reply":"2024-05-29T15:59:53.290075Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n train_df.select('click', 'banner_pos')\n .crosstab('banner_pos', 'click')\n .sort('banner_pos_click')\n .withColumn('click_rate', col('1') / (col('0') + col('1')))\n .show()\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T15:55:18.221207Z","iopub.execute_input":"2024-05-29T15:55:18.221876Z","iopub.status.idle":"2024-05-29T15:55:57.943273Z","shell.execute_reply.started":"2024-05-29T15:55:18.221828Z","shell.execute_reply":"2024-05-29T15:55:57.941826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.histplot(x='banner_pos', hue='click', data=train_df.select('banner_pos', 'click').toPandas(), multiple='stack', discrete=True)\ndistinct_values = train_df.select('banner_pos').distinct().sort('banner_pos').rdd.flatMap(lambda x: x).collect()\nax.set_xticks(distinct_values, distinct_values)\n\nplt.suptitle('Click Count by Banner Pos')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T15:56:47.681448Z","iopub.execute_input":"2024-05-29T15:56:47.683076Z","iopub.status.idle":"2024-05-29T15:57:58.795015Z","shell.execute_reply.started":"2024-05-29T15:56:47.683027Z","shell.execute_reply":"2024-05-29T15:57:58.793581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"(\n train_df.select('click', 'device_type')\n .crosstab('device_type', 'click')\n .sort('device_type_click')\n .withColumn('click_rate', col('1') / (col('0') + col('1')))\n .show()\n)","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:35:07.817938Z","iopub.execute_input":"2024-05-30T02:35:07.81897Z","iopub.status.idle":"2024-05-30T02:35:40.945248Z","shell.execute_reply.started":"2024-05-30T02:35:07.818932Z","shell.execute_reply":"2024-05-30T02:35:40.944183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ax = sns.histplot(x='device_type', hue='click', data=train_df.select('device_type', 'click').toPandas(), multiple='stack', discrete=True)\ndistinct_values = train_df.select('device_type').distinct().sort('device_type').rdd.flatMap(lambda x: x).collect()\nax.set_xticks(distinct_values, distinct_values)\n\nplt.suptitle('Click Count by Device Type')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-30T02:38:17.091171Z","iopub.execute_input":"2024-05-30T02:38:17.091568Z","iopub.status.idle":"2024-05-30T02:39:06.574244Z","shell.execute_reply.started":"2024-05-30T02:38:17.091536Z","shell.execute_reply":"2024-05-30T02:39:06.569915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by site id\ntrain_df.groupBy('site_id').count().orderBy('count', ascending=False).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:34:09.010931Z","iopub.execute_input":"2024-05-29T12:34:09.011425Z","iopub.status.idle":"2024-05-29T12:34:25.246586Z","shell.execute_reply.started":"2024-05-29T12:34:09.011383Z","shell.execute_reply":"2024-05-29T12:34:25.245554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by site domain\ntrain_df.groupBy('site_domain').count().orderBy('count', ascending=False).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:34:25.25147Z","iopub.execute_input":"2024-05-29T12:34:25.254175Z","iopub.status.idle":"2024-05-29T12:34:41.057663Z","shell.execute_reply.started":"2024-05-29T12:34:25.254127Z","shell.execute_reply":"2024-05-29T12:34:41.056561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by site category\ntrain_df.groupBy('site_category').count().orderBy('count', ascending=False).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:34:41.059491Z","iopub.execute_input":"2024-05-29T12:34:41.060361Z","iopub.status.idle":"2024-05-29T12:34:57.106095Z","shell.execute_reply.started":"2024-05-29T12:34:41.060318Z","shell.execute_reply":"2024-05-29T12:34:57.104996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by app id\ntrain_df.groupBy('app_id').count().orderBy('count', ascending=False).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:34:57.107752Z","iopub.execute_input":"2024-05-29T12:34:57.10808Z","iopub.status.idle":"2024-05-29T12:35:13.06199Z","shell.execute_reply.started":"2024-05-29T12:34:57.108054Z","shell.execute_reply":"2024-05-29T12:35:13.060853Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# group by app domain\ntrain_df.groupBy('app_domain').count().orderBy('count', ascending=False).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:35:13.069329Z","iopub.execute_input":"2024-05-29T12:35:13.070155Z","iopub.status.idle":"2024-05-29T12:35:28.692937Z","shell.execute_reply.started":"2024-05-29T12:35:13.070109Z","shell.execute_reply":"2024-05-29T12:35:28.691742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check missing value\nfrom pyspark.sql.functions import isnan, when, count, col\nprint(f'Number of missing value: {train_df.select([count(when(isnan(c) | col(c).isNull(), c)).alias(c) for c in train_df.columns]).collect()[0].asDict()}')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:35:28.696634Z","iopub.execute_input":"2024-05-29T12:35:28.697131Z","iopub.status.idle":"2024-05-29T12:38:10.537225Z","shell.execute_reply.started":"2024-05-29T12:35:28.697087Z","shell.execute_reply":"2024-05-29T12:38:10.536332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preprocessing","metadata":{}},{"cell_type":"code","source":"# print numerical and categorical columns\nnumerical_columns = [train_df.dtypes[i] for i in range(len(train_df.dtypes)) if train_df.dtypes[i][1] == 'int' or train_df.dtypes[i][1] == 'double']\ncategorical_columns = [train_df.dtypes[i] for i in range(len(train_df.dtypes)) if train_df.dtypes[i][1] == 'string']","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:10.538258Z","iopub.execute_input":"2024-05-29T12:38:10.538562Z","iopub.status.idle":"2024-05-29T12:38:10.546544Z","shell.execute_reply.started":"2024-05-29T12:38:10.538536Z","shell.execute_reply":"2024-05-29T12:38:10.545724Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"numerical_columns","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:10.547631Z","iopub.execute_input":"2024-05-29T12:38:10.548545Z","iopub.status.idle":"2024-05-29T12:38:10.563592Z","shell.execute_reply.started":"2024-05-29T12:38:10.548516Z","shell.execute_reply":"2024-05-29T12:38:10.562332Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"categorical_columns","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:10.565124Z","iopub.execute_input":"2024-05-29T12:38:10.566275Z","iopub.status.idle":"2024-05-29T12:38:10.581868Z","shell.execute_reply.started":"2024-05-29T12:38:10.566231Z","shell.execute_reply":"2024-05-29T12:38:10.580574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Label Encoding","metadata":{}},{"cell_type":"code","source":"click_count = train_df[train_df[\"click\"] == 1].count()\nno_click_count = train_df[train_df[\"click\"] == 0].count()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:10.583514Z","iopub.execute_input":"2024-05-29T12:38:10.584359Z","iopub.status.idle":"2024-05-29T12:38:40.216466Z","shell.execute_reply.started":"2024-05-29T12:38:10.584317Z","shell.execute_reply":"2024-05-29T12:38:40.214993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.title('Click vs. Non-Click')\nplt.bar(['Click', 'Non-Click'], [click_count, no_click_count])\nplt.ylabel('Count')\nplt.xlabel('Click')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:40.217953Z","iopub.execute_input":"2024-05-29T12:38:40.218536Z","iopub.status.idle":"2024-05-29T12:38:40.549827Z","shell.execute_reply.started":"2024-05-29T12:38:40.218492Z","shell.execute_reply":"2024-05-29T12:38:40.548442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def undersample_data (df, click_count, no_click_count) : \n fractions = click_count/no_click_count\n undersampled_df = df.sampleBy('click', fractions={0: fractions, 1: 1}, seed=11)\n \n return undersampled_df","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:40.551402Z","iopub.execute_input":"2024-05-29T12:38:40.552284Z","iopub.status.idle":"2024-05-29T12:38:40.558389Z","shell.execute_reply.started":"2024-05-29T12:38:40.552241Z","shell.execute_reply":"2024-05-29T12:38:40.557583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df = undersample_data(train_df, click_count, no_click_count)\nprint(f'Number of click: {preprocessed_df[preprocessed_df[\"click\"] == 1].count()}')\nprint(f'Number of non-click: {preprocessed_df[preprocessed_df[\"click\"] == 0].count()}')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:38:40.559757Z","iopub.execute_input":"2024-05-29T12:38:40.560117Z","iopub.status.idle":"2024-05-29T12:39:11.295388Z","shell.execute_reply.started":"2024-05-29T12:38:40.560087Z","shell.execute_reply":"2024-05-29T12:39:11.294494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Categorical Label Encoding","metadata":{}},{"cell_type":"code","source":"def labelEncoding (df) :\n categorical_columns = [df.dtypes[i] for i in range(len(df.dtypes)) if df.dtypes[i][1] == 'string']\n\n for col in categorical_columns:\n string_indexer = StringIndexer(inputCol=col[0], outputCol=col[0]+'_index')\n df = string_indexer.fit(df).transform(df)\n df = df.drop(col[0])\n\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:39:11.29635Z","iopub.execute_input":"2024-05-29T12:39:11.296659Z","iopub.status.idle":"2024-05-29T12:39:11.303283Z","shell.execute_reply.started":"2024-05-29T12:39:11.296631Z","shell.execute_reply":"2024-05-29T12:39:11.302313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df = labelEncoding(preprocessed_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:39:11.304962Z","iopub.execute_input":"2024-05-29T12:39:11.305336Z","iopub.status.idle":"2024-05-29T12:41:48.746213Z","shell.execute_reply.started":"2024-05-29T12:39:11.305307Z","shell.execute_reply":"2024-05-29T12:41:48.745097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show 10 rows of preprocessed df\npreprocessed_df.limit(10).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:41:48.747546Z","iopub.execute_input":"2024-05-29T12:41:48.748856Z","iopub.status.idle":"2024-05-29T12:42:29.489216Z","shell.execute_reply.started":"2024-05-29T12:41:48.748816Z","shell.execute_reply":"2024-05-29T12:42:29.48804Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Get Correlation Matrix","metadata":{}},{"cell_type":"code","source":"def get_corr (df) :\n vector_col = 'corr_features'\n assembler = VectorAssembler(inputCols=df.columns, outputCol=vector_col)\n df_vector = assembler.transform(df).select(vector_col)\n\n matrix = Correlation.corr(df_vector, vector_col).collect()[0][0]\n matrix = np.array(matrix.toArray())\n return matrix","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:42:29.490907Z","iopub.execute_input":"2024-05-29T12:42:29.491304Z","iopub.status.idle":"2024-05-29T12:42:29.498298Z","shell.execute_reply.started":"2024-05-29T12:42:29.491274Z","shell.execute_reply":"2024-05-29T12:42:29.496906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corr = get_corr(preprocessed_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:42:29.499484Z","iopub.execute_input":"2024-05-29T12:42:29.499792Z","iopub.status.idle":"2024-05-29T12:43:39.383319Z","shell.execute_reply.started":"2024-05-29T12:42:29.499766Z","shell.execute_reply":"2024-05-29T12:43:39.382105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plt.figure(figsize=(10, 10))\nplt.matshow(corr, fignum=1)\nplt.title('Correlation Heatmap')\nplt.xticks(range(len(preprocessed_df.columns)), preprocessed_df.columns, rotation='vertical')\nplt.yticks(range(len(preprocessed_df.columns)), preprocessed_df.columns)\nplt.colorbar()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:39.384614Z","iopub.execute_input":"2024-05-29T12:43:39.385065Z","iopub.status.idle":"2024-05-29T12:43:40.519852Z","shell.execute_reply.started":"2024-05-29T12:43:39.385028Z","shell.execute_reply":"2024-05-29T12:43:40.518788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Drop unnecessary columns","metadata":{}},{"cell_type":"code","source":"def drop_columns (df, collumns_to_drop) :\n for col in collumns_to_drop :\n df = df.drop(col)\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:40.521102Z","iopub.execute_input":"2024-05-29T12:43:40.521401Z","iopub.status.idle":"2024-05-29T12:43:40.527023Z","shell.execute_reply.started":"2024-05-29T12:43:40.521374Z","shell.execute_reply":"2024-05-29T12:43:40.525884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collumns_to_drop = ['id', 'datetime', 'C14']\npreprocessed_df = drop_columns(preprocessed_df, collumns_to_drop)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:40.52833Z","iopub.execute_input":"2024-05-29T12:43:40.528618Z","iopub.status.idle":"2024-05-29T12:43:40.570983Z","shell.execute_reply.started":"2024-05-29T12:43:40.528592Z","shell.execute_reply":"2024-05-29T12:43:40.569063Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df.printSchema()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:40.573804Z","iopub.execute_input":"2024-05-29T12:43:40.577904Z","iopub.status.idle":"2024-05-29T12:43:40.583891Z","shell.execute_reply.started":"2024-05-29T12:43:40.577851Z","shell.execute_reply":"2024-05-29T12:43:40.58307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Features Columns","metadata":{}},{"cell_type":"code","source":"def add_features_column (df) :\n columns = []\n \n for each in df.columns :\n if each != 'click' :\n columns.append(each)\n \n assembler = VectorAssembler(inputCols=columns, outputCol='features')\n df = assembler.transform(df)\n df = df.drop(*columns)\n \n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:40.585564Z","iopub.execute_input":"2024-05-29T12:43:40.586237Z","iopub.status.idle":"2024-05-29T12:43:40.598326Z","shell.execute_reply.started":"2024-05-29T12:43:40.586192Z","shell.execute_reply":"2024-05-29T12:43:40.595429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df = add_features_column(preprocessed_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:40.599947Z","iopub.execute_input":"2024-05-29T12:43:40.600877Z","iopub.status.idle":"2024-05-29T12:43:41.561913Z","shell.execute_reply.started":"2024-05-29T12:43:40.600841Z","shell.execute_reply":"2024-05-29T12:43:41.56047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df.limit(5).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:43:41.563933Z","iopub.execute_input":"2024-05-29T12:43:41.564428Z","iopub.status.idle":"2024-05-29T12:44:14.419043Z","shell.execute_reply.started":"2024-05-29T12:43:41.564388Z","shell.execute_reply":"2024-05-29T12:44:14.4181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Normalize Features","metadata":{}},{"cell_type":"code","source":"def normalize_features (df) :\n scaler = RobustScaler(inputCol='features', outputCol='scaled_features')\n df = scaler.fit(df).transform(df)\n df = df.drop('features')\n df = df.withColumnRenamed('scaled_features', 'features')\n return df","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:44:14.420207Z","iopub.execute_input":"2024-05-29T12:44:14.421136Z","iopub.status.idle":"2024-05-29T12:44:14.427667Z","shell.execute_reply.started":"2024-05-29T12:44:14.421098Z","shell.execute_reply":"2024-05-29T12:44:14.426576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df = normalize_features(preprocessed_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:44:14.429159Z","iopub.execute_input":"2024-05-29T12:44:14.429761Z","iopub.status.idle":"2024-05-29T12:45:05.254232Z","shell.execute_reply.started":"2024-05-29T12:44:14.429719Z","shell.execute_reply":"2024-05-29T12:45:05.253209Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preprocessed_df.limit(5).toPandas()","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:45:05.255235Z","iopub.execute_input":"2024-05-29T12:45:05.25555Z","iopub.status.idle":"2024-05-29T12:45:34.861173Z","shell.execute_reply.started":"2024-05-29T12:45:05.255515Z","shell.execute_reply":"2024-05-29T12:45:34.860101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Training","metadata":{}},{"cell_type":"markdown","source":"### Split Data","metadata":{}},{"cell_type":"code","source":"train_df, test_df = preprocessed_df.randomSplit([0.8, 0.2], seed=11)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:45:34.864076Z","iopub.execute_input":"2024-05-29T12:45:34.86445Z","iopub.status.idle":"2024-05-29T12:45:34.893272Z","shell.execute_reply.started":"2024-05-29T12:45:34.864419Z","shell.execute_reply":"2024-05-29T12:45:34.892134Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Number of train data: {train_df.count()}')\nprint(f'Number of test data: {test_df.count()}')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:45:34.894997Z","iopub.execute_input":"2024-05-29T12:45:34.896316Z","iopub.status.idle":"2024-05-29T12:47:10.381794Z","shell.execute_reply.started":"2024-05-29T12:45:34.896269Z","shell.execute_reply":"2024-05-29T12:47:10.380753Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Building","metadata":{}},{"cell_type":"code","source":"model = DecisionTreeClassifier(labelCol='click', featuresCol='features')\nmodel = model.fit(train_df)\nresult = model.transform(test_df)","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:47:10.382882Z","iopub.execute_input":"2024-05-29T12:47:10.383251Z","iopub.status.idle":"2024-05-29T12:50:21.090823Z","shell.execute_reply.started":"2024-05-29T12:47:10.38322Z","shell.execute_reply":"2024-05-29T12:50:21.089292Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Evaluation","metadata":{}},{"cell_type":"code","source":"evaluator = MulticlassClassificationEvaluator(labelCol='click', metricName='accuracy')\naccuracy = evaluator.evaluate(result)\nprecision = evaluator.evaluate(result, {evaluator.metricName: \"weightedPrecision\"})\nrecall = evaluator.evaluate(result, {evaluator.metricName: \"weightedRecall\"})\nf1 = evaluator.evaluate(result, {evaluator.metricName: \"f1\"})","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:50:21.101549Z","iopub.execute_input":"2024-05-29T12:50:21.102357Z","iopub.status.idle":"2024-05-29T12:53:33.881919Z","shell.execute_reply.started":"2024-05-29T12:50:21.102315Z","shell.execute_reply":"2024-05-29T12:53:33.880734Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f'Accuracy: {accuracy}')\nprint(f'Precision: {precision}')\nprint(f'Recall: {recall}')\nprint(f'F1: {f1}')","metadata":{"execution":{"iopub.status.busy":"2024-05-29T12:53:33.883057Z","iopub.execute_input":"2024-05-29T12:53:33.883373Z","iopub.status.idle":"2024-05-29T12:53:33.892236Z","shell.execute_reply.started":"2024-05-29T12:53:33.883347Z","shell.execute_reply":"2024-05-29T12:53:33.891095Z"},"trusted":true},"execution_count":null,"outputs":[]}]}