{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30698,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!sudo apt update\n!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n#Check this site for the latest download link https://www.apache.org/dyn/closer.lua/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz\n!wget -q https://dlcdn.apache.org/spark/spark-3.2.1/spark-3.2.1-bin-hadoop3.2.tgz\n!tar xf spark-3.2.1-bin-hadoop3.2.tgz\n!pip install -q findspark\n!pip install pyspark\n!pip install py4j\n\nimport os\nimport sys\nimport pandas as pd\nimport numpy as np\n# os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n# os.environ[\"SPARK_HOME\"] = \"/content/spark-3.2.1-bin-hadoop3.2\"\n\n\nimport findspark\nfindspark.init()\nfindspark.find()\n\nimport pyspark\n\nfrom pyspark.sql import DataFrame, SparkSession\nfrom typing import List\nimport pyspark.sql.types as T\nimport pyspark.sql.functions as F\n\nspark= SparkSession \\\n       .builder \\\n       .appName(\"Our CTR Prediction\") \\\n       .config(\"spark.executor.memory\", \"20G\") \\\n       .config(\"spark.executor.memoryOverhead\", \"4G\") \\\n       .getOrCreate()\n\nspark\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-21T14:29:09.493644Z","iopub.execute_input":"2024-05-21T14:29:09.493990Z","iopub.status.idle":"2024-05-21T14:30:47.028152Z","shell.execute_reply.started":"2024-05-21T14:29:09.493960Z","shell.execute_reply":"2024-05-21T14:30:47.026998Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ndf = spark.read.format('csv').options(header=True,inferSchema=True).load('/kaggle/input/avazu-ctr-prediction/train.gz')","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:31:13.229725Z","iopub.execute_input":"2024-05-21T14:31:13.230381Z","iopub.status.idle":"2024-05-21T14:34:30.824365Z","shell.execute_reply.started":"2024-05-21T14:31:13.230330Z","shell.execute_reply":"2024-05-21T14:34:30.823378Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.show(30, truncate=50)","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:35:30.493608Z","iopub.execute_input":"2024-05-21T14:35:30.494580Z","iopub.status.idle":"2024-05-21T14:35:31.058410Z","shell.execute_reply.started":"2024-05-21T14:35:30.494545Z","shell.execute_reply":"2024-05-21T14:35:31.057457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Export to parguet","metadata":{}},{"cell_type":"code","source":"rp = df.repartition(40)\nrp .write.parquet(\"ctr.parquet\")  # You can use ORC instead of Parquet\n","metadata":{"scrolled":true,"execution":{"iopub.status.busy":"2024-05-21T14:36:14.337768Z","iopub.execute_input":"2024-05-21T14:36:14.338466Z","iopub.status.idle":"2024-05-21T14:42:56.645918Z","shell.execute_reply.started":"2024-05-21T14:36:14.338432Z","shell.execute_reply":"2024-05-21T14:42:56.644760Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = spark.read.parquet(\"ctr.parquet\").rdd.repartition(400)\n","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:46:03.104966Z","iopub.execute_input":"2024-05-21T14:46:03.106095Z","iopub.status.idle":"2024-05-21T14:46:03.527123Z","shell.execute_reply.started":"2024-05-21T14:46:03.106056Z","shell.execute_reply":"2024-05-21T14:46:03.525845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Jumlah data ada 40 mil","metadata":{}},{"cell_type":"code","source":"df.count()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T14:46:08.805100Z","iopub.execute_input":"2024-05-21T14:46:08.805784Z","iopub.status.idle":"2024-05-21T15:00:20.384088Z","shell.execute_reply.started":"2024-05-21T14:46:08.805751Z","shell.execute_reply":"2024-05-21T15:00:20.383152Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.printSchema()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:12:45.252687Z","iopub.execute_input":"2024-05-20T10:12:45.253054Z","iopub.status.idle":"2024-05-20T10:12:45.262413Z","shell.execute_reply.started":"2024-05-20T10:12:45.253029Z","shell.execute_reply":"2024-05-20T10:12:45.261144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pyspark.sql.types import *\nfrom pyspark.sql.functions import *","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:15:33.232390Z","iopub.execute_input":"2024-05-21T13:15:33.233008Z","iopub.status.idle":"2024-05-21T13:15:33.240127Z","shell.execute_reply.started":"2024-05-21T13:15:33.232973Z","shell.execute_reply":"2024-05-21T13:15:33.237918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['hour'].isnull().sum().sum()\ndf.select([count(when(isnan(c) | isnull(c), 1)).alias(c) for c in large_df.columns]).show()","metadata":{"execution":{"iopub.status.busy":"2024-05-20T10:14:11.161037Z","iopub.execute_input":"2024-05-20T10:14:11.161845Z","iopub.status.idle":"2024-05-20T10:28:36.528688Z","shell.execute_reply.started":"2024-05-20T10:14:11.161811Z","shell.execute_reply":"2024-05-20T10:28:36.527540Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## ambil 1 million dulu","metadata":{}},{"cell_type":"code","source":"spark.stop()","metadata":{"execution":{"iopub.status.busy":"2024-05-21T13:02:19.744860Z","iopub.execute_input":"2024-05-21T13:02:19.745567Z","iopub.status.idle":"2024-05-21T13:02:20.544337Z","shell.execute_reply.started":"2024-05-21T13:02:19.745524Z","shell.execute_reply":"2024-05-21T13:02:20.543215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}