{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":4120,"databundleVersionId":44256,"sourceType":"competition"}],"dockerImageVersionId":30715,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!pip install pyspark","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-06-03T12:45:31.336479Z","iopub.execute_input":"2024-06-03T12:45:31.337365Z","iopub.status.idle":"2024-06-03T12:46:26.256955Z","shell.execute_reply.started":"2024-06-03T12:45:31.337325Z","shell.execute_reply":"2024-06-03T12:46:26.255402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nfrom pyspark.sql import SparkSession\nfrom multiprocessing import cpu_count","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:46:44.937931Z","iopub.execute_input":"2024-06-03T12:46:44.938412Z","iopub.status.idle":"2024-06-03T12:46:44.945254Z","shell.execute_reply.started":"2024-06-03T12:46:44.938376Z","shell.execute_reply":"2024-06-03T12:46:44.943671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cpu_count()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:46:46.236546Z","iopub.execute_input":"2024-06-03T12:46:46.236939Z","iopub.status.idle":"2024-06-03T12:46:46.245264Z","shell.execute_reply.started":"2024-06-03T12:46:46.236907Z","shell.execute_reply":"2024-06-03T12:46:46.243923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"spark = (\n    SparkSession.builder\n    .master('local[*]')\n    .config('spark.driver.memory', '4g')\n    .appName('ctr')\n    .getOrCreate()\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:46:26.385770Z","iopub.execute_input":"2024-06-03T12:46:26.386103Z","iopub.status.idle":"2024-06-03T12:46:32.179058Z","shell.execute_reply.started":"2024-06-03T12:46:26.386075Z","shell.execute_reply":"2024-06-03T12:46:32.177527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glob.glob('/kaggle/input/**/**')","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:46:55.045977Z","iopub.execute_input":"2024-06-03T12:46:55.046374Z","iopub.status.idle":"2024-06-03T12:46:55.055608Z","shell.execute_reply.started":"2024-06-03T12:46:55.046346Z","shell.execute_reply":"2024-06-03T12:46:55.054502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = (\n    spark.read\n    .option('header', True)\n    .csv('/kaggle/input/avazu-ctr-prediction/train.gz')\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:46:56.094420Z","iopub.execute_input":"2024-06-03T12:46:56.095140Z","iopub.status.idle":"2024-06-03T12:47:03.940734Z","shell.execute_reply.started":"2024-06-03T12:46:56.095087Z","shell.execute_reply":"2024-06-03T12:47:03.939527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.show(3)","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:47:07.215629Z","iopub.execute_input":"2024-06-03T12:47:07.216738Z","iopub.status.idle":"2024-06-03T12:47:08.216774Z","shell.execute_reply.started":"2024-06-03T12:47:07.216696Z","shell.execute_reply":"2024-06-03T12:47:08.215592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.rdd.getNumPartitions()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:47:09.451634Z","iopub.execute_input":"2024-06-03T12:47:09.452605Z","iopub.status.idle":"2024-06-03T12:47:09.545790Z","shell.execute_reply.started":"2024-06-03T12:47:09.452568Z","shell.execute_reply":"2024-06-03T12:47:09.544571Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.printSchema()","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:47:10.898959Z","iopub.execute_input":"2024-06-03T12:47:10.899955Z","iopub.status.idle":"2024-06-03T12:47:10.909319Z","shell.execute_reply.started":"2024-06-03T12:47:10.899915Z","shell.execute_reply":"2024-06-03T12:47:10.908030Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"partitioned_df = (\n    df.repartition(4).write\n    .option('header', True)\n    .parquet('/kaggle/working/ctr', mode='overwrite')\n)","metadata":{"execution":{"iopub.status.busy":"2024-06-03T12:58:42.122210Z","iopub.execute_input":"2024-06-03T12:58:42.122754Z","iopub.status.idle":"2024-06-03T13:07:46.177611Z","shell.execute_reply.started":"2024-06-03T12:58:42.122712Z","shell.execute_reply":"2024-06-03T13:07:46.175950Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}