{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.12","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10038,"databundleVersionId":180088,"sourceType":"competition"}],"dockerImageVersionId":30918,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:21:10.148360Z","iopub.execute_input":"2025-03-17T09:21:10.148617Z","iopub.status.idle":"2025-03-17T09:21:10.156829Z","shell.execute_reply.started":"2025-03-17T09:21:10.148595Z","shell.execute_reply":"2025-03-17T09:21:10.155637Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"!pip install vaex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:21:10.157778Z","iopub.execute_input":"2025-03-17T09:21:10.158208Z","iopub.status.idle":"2025-03-17T09:21:27.023803Z","shell.execute_reply.started":"2025-03-17T09:21:10.158166Z","shell.execute_reply":"2025-03-17T09:21:27.022220Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import vaex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:21:27.025318Z","iopub.execute_input":"2025-03-17T09:21:27.025806Z","iopub.status.idle":"2025-03-17T09:21:28.828870Z","shell.execute_reply.started":"2025-03-17T09:21:27.025759Z","shell.execute_reply":"2025-03-17T09:21:28.827738Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndf = vaex.open(\"/kaggle/input/ga-customer-revenue-prediction/train_v2.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:21:28.829740Z","iopub.execute_input":"2025-03-17T09:21:28.830247Z","iopub.status.idle":"2025-03-17T09:22:46.675965Z","shell.execute_reply.started":"2025-03-17T09:21:28.830220Z","shell.execute_reply":"2025-03-17T09:22:46.674577Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import pandas as pd","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-16T19:11:56.169375Z","iopub.execute_input":"2025-03-16T19:11:56.169754Z","iopub.status.idle":"2025-03-16T19:11:56.174069Z","shell.execute_reply.started":"2025-03-16T19:11:56.169701Z","shell.execute_reply":"2025-03-16T19:11:56.173172Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"%%time\n\ndf_p = pd.read_csv(\"/kaggle/input/ga-customer-revenue-prediction/train_v2.csv\")","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-14T09:18:54.220999Z","iopub.execute_input":"2025-03-14T09:18:54.221343Z","iopub.status.idle":"2025-03-14T09:26:17.394879Z","shell.execute_reply.started":"2025-03-14T09:18:54.221318Z","shell.execute_reply":"2025-03-14T09:26:17.393567Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:22:46.677459Z","iopub.execute_input":"2025-03-17T09:22:46.677953Z","iopub.status.idle":"2025-03-17T09:22:53.017231Z","shell.execute_reply.started":"2025-03-17T09:22:46.677901Z","shell.execute_reply":"2025-03-17T09:22:53.016097Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"def extract_revenue(s):\n    \"\"\"Parse transactionRevenue from JSON-like string\"\"\"\n    if 'transactionRevenue' in s:\n        try:\n            # Extract value after \"transactionRevenue\": \n            value_part = s.split('transactionRevenue\":')[1].split(',')[0]\n            return float(value_part.strip(' \"'))  # Remove quotes/whitespace\n        except:\n            return 0.0\n    return 0.0\n\n# Apply function to totals column (lazily)\ndf['transactionRevenue'] = df.totals.apply(extract_revenue)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:23:06.110216Z","iopub.execute_input":"2025-03-17T09:23:06.110621Z","iopub.status.idle":"2025-03-17T09:23:06.117713Z","shell.execute_reply.started":"2025-03-17T09:23:06.110592Z","shell.execute_reply":"2025-03-17T09:23:06.116082Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import numpy as np\n\ndf['tx_log'] = np.log(df.transactionRevenue + 1)\n\ndf['visitStartTime_dt'] = df.visitStartTime.astype('datetime64[s]')  # 's' for epoch seconds\ndf['visit_hour'] = df.visitStartTime_dt.dt.hour  # Extract hour\n\nprint(\"Virtual columns created:\")\nprint(df.tx_log.expression)\nprint(df.visit_hour.expression)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:23:12.678601Z","iopub.execute_input":"2025-03-17T09:23:12.679058Z","iopub.status.idle":"2025-03-17T09:23:12.689348Z","shell.execute_reply.started":"2025-03-17T09:23:12.679014Z","shell.execute_reply":"2025-03-17T09:23:12.688206Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df['is_high_value'] = df.transactionRevenue > 1000","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:23:15.101964Z","iopub.execute_input":"2025-03-17T09:23:15.102388Z","iopub.status.idle":"2025-03-17T09:23:15.107827Z","shell.execute_reply.started":"2025-03-17T09:23:15.102360Z","shell.execute_reply":"2025-03-17T09:23:15.106562Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_ex = vaex.example()\nlimits = [-10, 10]\n\ndelayed_count = df_ex.count(df_ex.y, binby=df_ex.x, limits=limits, shape=4, delay=True)\ndelayed_sum = df_ex.sum(df_ex.y, binby=df_ex.x, limits=limits, shape=4, delay=True)\n\n@vaex.delayed\ndef calculate_mean(sums, counts):\n    print('calculating mean')\n    return sums / counts\n\nprint('before calling mean')\n# since calculate_mean is decorated with vaex.delayed\n# this now also returns a 'delayed' object (a promise)\ndelayed_mean = calculate_mean(delayed_sum, delayed_count)\n\n@vaex.delayed\ndef print_mean(means):\n    print('means', means)\nprint_mean(delayed_mean)\n\nprint('before calling execute')\ndf_ex.execute()  # Execute all delayed computations\n\n# Retrieve the result after execution\nmeans = delayed_mean.get()\nprint('same means', means)","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:23:39.373341Z","iopub.execute_input":"2025-03-17T09:23:39.373781Z","iopub.status.idle":"2025-03-17T09:23:40.438318Z","shell.execute_reply.started":"2025-03-17T09:23:39.373751Z","shell.execute_reply":"2025-03-17T09:23:40.437231Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_ex.viz.histogram(df_ex.x, limits='99.7%')","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:23:52.626288Z","iopub.execute_input":"2025-03-17T09:23:52.626806Z","iopub.status.idle":"2025-03-17T09:23:52.993171Z","shell.execute_reply.started":"2025-03-17T09:23:52.626769Z","shell.execute_reply":"2025-03-17T09:23:52.992020Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"import vaex.jupyter\nimport matplotlib.pyplot as plt","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:27:16.779329Z","iopub.execute_input":"2025-03-17T09:27:16.779811Z","iopub.status.idle":"2025-03-17T09:27:16.977704Z","shell.execute_reply.started":"2025-03-17T09:27:16.779764Z","shell.execute_reply":"2025-03-17T09:27:16.976199Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_ex","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:27:57.065231Z","iopub.execute_input":"2025-03-17T09:27:57.065653Z","iopub.status.idle":"2025-03-17T09:27:57.088375Z","shell.execute_reply.started":"2025-03-17T09:27:57.065623Z","shell.execute_reply":"2025-03-17T09:27:57.087121Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"vaex.multithreading.thread_count_default = 8\nimport vaex.ml","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:35:57.776886Z","iopub.execute_input":"2025-03-17T09:35:57.777482Z","iopub.status.idle":"2025-03-17T09:35:57.886714Z","shell.execute_reply.started":"2025-03-17T09:35:57.777426Z","shell.execute_reply":"2025-03-17T09:35:57.885180Z"}},"outputs":[],"execution_count":null},{"cell_type":"code","source":"df_iris = vaex.datasets.iris()\nfeatures = ['petal_length', 'petal_width', 'sepal_length', 'sepal_width']\nscaler = vaex.ml.StandardScaler(features=features, prefix='scaled_')\nscaler.fit(df_iris)\ndf_trans = scaler.transform(df_iris)\ndf_trans","metadata":{"trusted":true,"execution":{"iopub.status.busy":"2025-03-17T09:36:10.101585Z","iopub.execute_input":"2025-03-17T09:36:10.102019Z","iopub.status.idle":"2025-03-17T09:36:10.304698Z","shell.execute_reply.started":"2025-03-17T09:36:10.101968Z","shell.execute_reply":"2025-03-17T09:36:10.303036Z"}},"outputs":[],"execution_count":null}]}