{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.13","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"none","dataSources":[{"sourceId":10038,"databundleVersionId":180088,"sourceType":"competition"}],"dockerImageVersionId":30673,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":false}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2024-05-13T14:30:11.509493Z","iopub.execute_input":"2024-05-13T14:30:11.509892Z","iopub.status.idle":"2024-05-13T14:30:12.707026Z","shell.execute_reply.started":"2024-05-13T14:30:11.509846Z","shell.execute_reply":"2024-05-13T14:30:12.705911Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport json\nimport numpy as np \nimport pandas as pd\nimport seaborn as sns \nimport matplotlib as mpl\nimport scipy.stats as st\nimport plotly.express as px\nimport plotly.graph_objs as go\nimport matplotlib.pyplot as plt\nimport matplotlib.ticker as ticker\n\nfrom tqdm import tqdm\nfrom plotly import tools\nfrom pandas import json_normalize\nfrom plotly.offline import init_notebook_mode, iplot\nimport xgboost as xgb\nfrom sklearn.metrics import mean_squared_error\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\n# Create a list of colors \ncolors = ['b', 'g', 'r', 'c', 'm', 'y', 'k']\n\nsns.set_style('whitegrid')\nplt.rcParams['figure.figsize'] = 10, 6\npd.options.display.float_format = '{:,.0f}'.format\nnp.set_printoptions(formatter={'float': '{: 0.0f}'.format})","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:30:12.709336Z","iopub.execute_input":"2024-05-13T14:30:12.710230Z","iopub.status.idle":"2024-05-13T14:30:15.537968Z","shell.execute_reply.started":"2024-05-13T14:30:12.710190Z","shell.execute_reply":"2024-05-13T14:30:15.536795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Read csv and flatten json fields:   \nhttps://www.kaggle.com/julian3833/1-quick-start-read-csv-and-flatten-json-fields\n\n- Take string as an input and create a json object with the specified string as object name   \nhttps://towardsdatascience.com/why-and-how-to-use-pandas-with-large-data-9594dda2ea4c","metadata":{}},{"cell_type":"code","source":"\n# Extract features from 'device', 'geoNetwork', 'totals', 'trafficSource’. \njson_columns = ['device', 'geoNetwork','totals', 'trafficSource']\n\ndef load_dataframe(csv_path):\n    path = csv_path\n    df = pd.read_csv(path, converters={column: json.loads for column in json_columns}, \n                     dtype={'fullVisitorId': 'str'})\n   \n    for column in json_columns:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n\n    return df\n\n","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:30:15.539243Z","iopub.execute_input":"2024-05-13T14:30:15.539570Z","iopub.status.idle":"2024-05-13T14:30:15.549271Z","shell.execute_reply.started":"2024-05-13T14:30:15.539542Z","shell.execute_reply":"2024-05-13T14:30:15.547244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = load_dataframe('/kaggle/input/ga-customer-revenue-prediction/train.csv')\ntest = load_dataframe('/kaggle/input/ga-customer-revenue-prediction/test.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:30:15.552107Z","iopub.execute_input":"2024-05-13T14:30:15.552437Z","iopub.status.idle":"2024-05-13T14:33:49.485470Z","shell.execute_reply.started":"2024-05-13T14:30:15.552411Z","shell.execute_reply":"2024-05-13T14:33:49.484243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Store the dataframes as csv files\ntrain.to_csv('train_df.csv')\ntest.to_csv('test_df.csv')","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:33:49.487206Z","iopub.execute_input":"2024-05-13T14:33:49.487637Z","iopub.status.idle":"2024-05-13T14:35:29.755791Z","shell.execute_reply.started":"2024-05-13T14:33:49.487600Z","shell.execute_reply":"2024-05-13T14:35:29.754216Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Read csv files\ntrain = pd.read_csv('train_df.csv', dtype={'fullVisitorId': 'str'}, index_col=0)\n\ntest = pd.read_csv('test_df.csv', dtype={'fullVisitorId': 'str'}, index_col=0)\n\nprint(train.shape)\nprint(test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:35:29.757822Z","iopub.execute_input":"2024-05-13T14:35:29.758229Z","iopub.status.idle":"2024-05-13T14:36:07.323010Z","shell.execute_reply.started":"2024-05-13T14:35:29.758199Z","shell.execute_reply":"2024-05-13T14:36:07.321379Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:07.324900Z","iopub.execute_input":"2024-05-13T14:36:07.326207Z","iopub.status.idle":"2024-05-13T14:36:07.361498Z","shell.execute_reply.started":"2024-05-13T14:36:07.326169Z","shell.execute_reply":"2024-05-13T14:36:07.360150Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:07.363505Z","iopub.execute_input":"2024-05-13T14:36:07.363831Z","iopub.status.idle":"2024-05-13T14:36:11.114096Z","shell.execute_reply.started":"2024-05-13T14:36:07.363805Z","shell.execute_reply":"2024-05-13T14:36:11.112952Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:11.115643Z","iopub.execute_input":"2024-05-13T14:36:11.116608Z","iopub.status.idle":"2024-05-13T14:36:11.142309Z","shell.execute_reply.started":"2024-05-13T14:36:11.116568Z","shell.execute_reply":"2024-05-13T14:36:11.141157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:11.147183Z","iopub.execute_input":"2024-05-13T14:36:11.147969Z","iopub.status.idle":"2024-05-13T14:36:14.422955Z","shell.execute_reply.started":"2024-05-13T14:36:11.147931Z","shell.execute_reply":"2024-05-13T14:36:14.421592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 2. Data Preprocessing","metadata":{}},{"cell_type":"markdown","source":" 2.1 Remove Constant Features\n\nFeatures that have only one unique value are often referred to as constant features or constant columns.   \n\nThese features do not provide any variability in the data, and as a result, they typically do not contribute    \nuseful information to the model. We might as well drop these features.","metadata":{}},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:14.424876Z","iopub.execute_input":"2024-05-13T14:36:14.425211Z","iopub.status.idle":"2024-05-13T14:36:14.433371Z","shell.execute_reply.started":"2024-05-13T14:36:14.425177Z","shell.execute_reply":"2024-05-13T14:36:14.432153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Keep 'totals_bounces' column\ntrain['totals_bounces'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:14.435112Z","iopub.execute_input":"2024-05-13T14:36:14.435466Z","iopub.status.idle":"2024-05-13T14:36:14.454759Z","shell.execute_reply.started":"2024-05-13T14:36:14.435437Z","shell.execute_reply":"2024-05-13T14:36:14.453908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['totals_bounces'].fillna(0, inplace=True)\ntrain['totals_bounces'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:14.456497Z","iopub.execute_input":"2024-05-13T14:36:14.459183Z","iopub.status.idle":"2024-05-13T14:36:14.484816Z","shell.execute_reply.started":"2024-05-13T14:36:14.459140Z","shell.execute_reply":"2024-05-13T14:36:14.483524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check unique values in train columns\ntrain.nunique()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:14.486080Z","iopub.execute_input":"2024-05-13T14:36:14.486405Z","iopub.status.idle":"2024-05-13T14:36:18.890427Z","shell.execute_reply.started":"2024-05-13T14:36:14.486378Z","shell.execute_reply":"2024-05-13T14:36:18.889385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identify constant features\nconstant_features = train.columns[train.nunique() == 1]\n\n# Remove constant features\ntrain.drop(columns=constant_features, inplace=True)\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:18.892283Z","iopub.execute_input":"2024-05-13T14:36:18.892595Z","iopub.status.idle":"2024-05-13T14:36:23.466920Z","shell.execute_reply.started":"2024-05-13T14:36:18.892568Z","shell.execute_reply":"2024-05-13T14:36:23.465685Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Keep 'totals_bounces' column\ntest['totals_bounces'].unique()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:23.468601Z","iopub.execute_input":"2024-05-13T14:36:23.469319Z","iopub.status.idle":"2024-05-13T14:36:23.485954Z","shell.execute_reply.started":"2024-05-13T14:36:23.469280Z","shell.execute_reply":"2024-05-13T14:36:23.484712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['totals_bounces'].fillna(0, inplace=True)\ntest['totals_bounces'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:23.487242Z","iopub.execute_input":"2024-05-13T14:36:23.487570Z","iopub.status.idle":"2024-05-13T14:36:23.510175Z","shell.execute_reply.started":"2024-05-13T14:36:23.487543Z","shell.execute_reply":"2024-05-13T14:36:23.508931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check unique values in test columns\ntest.nunique()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:23.511631Z","iopub.execute_input":"2024-05-13T14:36:23.512268Z","iopub.status.idle":"2024-05-13T14:36:27.473202Z","shell.execute_reply.started":"2024-05-13T14:36:23.512237Z","shell.execute_reply":"2024-05-13T14:36:27.471999Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Identify constant features\nconstant_features = test.columns[test.nunique() == 1]\n\n# Remove constant features\ntest.drop(columns=constant_features, inplace=True)\ntest.shape","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:27.474346Z","iopub.execute_input":"2024-05-13T14:36:27.474677Z","iopub.status.idle":"2024-05-13T14:36:31.665313Z","shell.execute_reply.started":"2024-05-13T14:36:27.474636Z","shell.execute_reply":"2024-05-13T14:36:31.664094Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.2 Find Percentage of missing values in the columns\n\n- credits:   \nhttps://stackoverflow.com/questions/51070985/find-out-the-percentage-of-missing-values-in-each-column-in-the-given-dataset","metadata":{}},{"cell_type":"code","source":"# train_df - Check the percentage of missing values in each column\npercent_missing_train = train.isnull().sum() * 100 / len(train)\n\nmissing_value_train = pd.DataFrame({'percent_missing': percent_missing_train})\nmissing_value_train.sort_values(by='percent_missing', ascending=False, inplace=True)\n\n# Columns with percent_missing > 0\nmissing_value_train_filtered = missing_value_train.loc[missing_value_train['percent_missing'] > 0]\nmissing_value_train_filtered","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:31.666884Z","iopub.execute_input":"2024-05-13T14:36:31.667332Z","iopub.status.idle":"2024-05-13T14:36:33.516411Z","shell.execute_reply.started":"2024-05-13T14:36:31.667291Z","shell.execute_reply":"2024-05-13T14:36:33.515172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# test_df - Check the percentage of missing values in each column\npercent_missing_test = test.isnull().sum() * 100 / len(test)\n\nmissing_value_test = pd.DataFrame({#'column_name': test_df.columns,\n                                 'percent_missing': percent_missing_test})\nmissing_value_test.sort_values(by='percent_missing', ascending=False, inplace=True)\n\n# Columns with percent_missing > 0\nmissing_value_test_filtered = missing_value_test.loc[missing_value_test['percent_missing'] > 0]\nmissing_value_test_filtered","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:33.517763Z","iopub.execute_input":"2024-05-13T14:36:33.518120Z","iopub.status.idle":"2024-05-13T14:36:35.178173Z","shell.execute_reply.started":"2024-05-13T14:36:33.518092Z","shell.execute_reply":"2024-05-13T14:36:35.176814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.3 Columns to Drop","metadata":{}},{"cell_type":"code","source":"train.columns","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:35.179532Z","iopub.execute_input":"2024-05-13T14:36:35.179896Z","iopub.status.idle":"2024-05-13T14:36:35.188232Z","shell.execute_reply.started":"2024-05-13T14:36:35.179866Z","shell.execute_reply":"2024-05-13T14:36:35.187012Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cols_to_drop = ['geoNetwork_continent', 'geoNetwork_subContinent', \n                'geoNetwork_metro', 'geoNetwork_city', 'geoNetwork_networkDomain', \n                'trafficSource_campaign', 'trafficSource_keyword',\n                'trafficSource_referralPath', 'trafficSource_adwordsClickInfo.page',\n                'trafficSource_adwordsClickInfo.slot', 'trafficSource_adwordsClickInfo.gclId',\n                'trafficSource_adContent', 'visitId', 'visitStartTime']","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:35.189383Z","iopub.execute_input":"2024-05-13T14:36:35.189694Z","iopub.status.idle":"2024-05-13T14:36:35.197995Z","shell.execute_reply.started":"2024-05-13T14:36:35.189668Z","shell.execute_reply":"2024-05-13T14:36:35.196731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train.copy()\n\ndf_train = df_train.drop(columns=cols_to_drop, axis=1)\ndf_train","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:35.199311Z","iopub.execute_input":"2024-05-13T14:36:35.199645Z","iopub.status.idle":"2024-05-13T14:36:35.703045Z","shell.execute_reply.started":"2024-05-13T14:36:35.199604Z","shell.execute_reply":"2024-05-13T14:36:35.701903Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:35.704955Z","iopub.execute_input":"2024-05-13T14:36:35.705397Z","iopub.status.idle":"2024-05-13T14:36:36.745256Z","shell.execute_reply.started":"2024-05-13T14:36:35.705359Z","shell.execute_reply":"2024-05-13T14:36:36.743915Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.describe().T","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:36.746763Z","iopub.execute_input":"2024-05-13T14:36:36.747229Z","iopub.status.idle":"2024-05-13T14:36:36.949772Z","shell.execute_reply.started":"2024-05-13T14:36:36.747189Z","shell.execute_reply":"2024-05-13T14:36:36.948574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = test.copy()\n\ndf_test = df_test.drop(columns=cols_to_drop, axis=1)\ndf_test","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:36.951624Z","iopub.execute_input":"2024-05-13T14:36:36.952401Z","iopub.status.idle":"2024-05-13T14:36:37.374045Z","shell.execute_reply.started":"2024-05-13T14:36:36.952359Z","shell.execute_reply":"2024-05-13T14:36:37.372909Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.info()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:37.386777Z","iopub.execute_input":"2024-05-13T14:36:37.388109Z","iopub.status.idle":"2024-05-13T14:36:38.342696Z","shell.execute_reply.started":"2024-05-13T14:36:37.388053Z","shell.execute_reply":"2024-05-13T14:36:38.341443Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test.describe().T","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:38.343989Z","iopub.execute_input":"2024-05-13T14:36:38.344294Z","iopub.status.idle":"2024-05-13T14:36:38.476027Z","shell.execute_reply.started":"2024-05-13T14:36:38.344268Z","shell.execute_reply":"2024-05-13T14:36:38.475095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 2.4 Impute Missing Values","metadata":{}},{"cell_type":"code","source":"# numerical columns fillna\ndf_train['totals_pageviews'].fillna(0, inplace=True)\ndf_train['totals_transactionRevenue'].fillna(0, inplace=True)\n\ndf_test['totals_pageviews'].fillna(0, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:38.477459Z","iopub.execute_input":"2024-05-13T14:36:38.478417Z","iopub.status.idle":"2024-05-13T14:36:38.494373Z","shell.execute_reply.started":"2024-05-13T14:36:38.478384Z","shell.execute_reply":"2024-05-13T14:36:38.492990Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.columns","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:38.495958Z","iopub.execute_input":"2024-05-13T14:36:38.496317Z","iopub.status.idle":"2024-05-13T14:36:38.504777Z","shell.execute_reply.started":"2024-05-13T14:36:38.496287Z","shell.execute_reply":"2024-05-13T14:36:38.503444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Replace 'nan', 'NaN', '(not set)' in both dataframes\ncolumns_to_process = ['device_browser', 'device_operatingSystem', 'trafficSource_source', \n                      'trafficSource_medium', 'trafficSource_adwordsClickInfo.adNetworkType']\n\nvalues_to_replace = ['nan', 'NaN', '(not set)', np.nan]\nreplacement_value = 'others'\n\nfor column in columns_to_process:\n    df_train[column].replace(values_to_replace, replacement_value, inplace=True)\n\nfor column in columns_to_process:\n    df_test[column].replace(values_to_replace, replacement_value, inplace=True)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:38.506445Z","iopub.execute_input":"2024-05-13T14:36:38.506831Z","iopub.status.idle":"2024-05-13T14:36:40.463575Z","shell.execute_reply.started":"2024-05-13T14:36:38.506792Z","shell.execute_reply":"2024-05-13T14:36:40.462362Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train['trafficSource_adwordsClickInfo.adNetworkType'].unique()\n","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:40.465223Z","iopub.execute_input":"2024-05-13T14:36:40.465565Z","iopub.status.idle":"2024-05-13T14:36:40.545520Z","shell.execute_reply.started":"2024-05-13T14:36:40.465536Z","shell.execute_reply":"2024-05-13T14:36:40.544347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Find the mismatched columns and the target variable\ntrain_columns = set(df_train.columns)\ntest_columns = set(df_test.columns)\nmismatched_columns = train_columns.symmetric_difference(test_columns)\n\nprint(\"Columns that don't match:\")\nprint(mismatched_columns)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:40.548016Z","iopub.execute_input":"2024-05-13T14:36:40.548471Z","iopub.status.idle":"2024-05-13T14:36:40.557298Z","shell.execute_reply.started":"2024-05-13T14:36:40.548431Z","shell.execute_reply":"2024-05-13T14:36:40.555954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3. 'channelGrouping' & 'device' columns","metadata":{}},{"cell_type":"code","source":"device_columns = ['channelGrouping', 'device_browser', 'device_operatingSystem', \n                  'device_isMobile', 'device_deviceCategory']\n\nfig, ax = plt.subplots(figsize=(15, 8))\n\nfor i, col in enumerate(device_columns, 1):\n    plt.subplot(2, 3, i)\n    \n    ax = sns.countplot(y=df_train[col], order=df_train[col].value_counts()[:10].index, )\n    plt.title(f'Count of {col}')\n\n    total = len(df_train[col])\n\n    for c in ax.containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        ax.bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.tight_layout()\nplt.show()\n\n# Explicitly close the figure to remove overlapping axes\nplt.close(fig)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:40.559073Z","iopub.execute_input":"2024-05-13T14:36:40.559516Z","iopub.status.idle":"2024-05-13T14:36:45.921080Z","shell.execute_reply.started":"2024-05-13T14:36:40.559474Z","shell.execute_reply":"2024-05-13T14:36:45.920098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.1 Channels","metadata":{}},{"cell_type":"code","source":"# Define the aggregation functions for each column\naggregations = {'fullVisitorId': 'count',\n                'totals_pageviews': 'sum',\n                'totals_transactionRevenue': 'sum'\n}","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:45.922586Z","iopub.execute_input":"2024-05-13T14:36:45.923198Z","iopub.status.idle":"2024-05-13T14:36:45.928211Z","shell.execute_reply.started":"2024-05-13T14:36:45.923166Z","shell.execute_reply":"2024-05-13T14:36:45.926959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_channels = df_train.groupby('channelGrouping').agg(aggregations\n                                ).sort_values(by='totals_transactionRevenue', ascending=False)\n\ndf_channels","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:45.929768Z","iopub.execute_input":"2024-05-13T14:36:45.930225Z","iopub.status.idle":"2024-05-13T14:36:46.173542Z","shell.execute_reply.started":"2024-05-13T14:36:45.930195Z","shell.execute_reply":"2024-05-13T14:36:46.172377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_channels.columns):\n    df_channels[col].sort_values().plot(kind='barh', ax=axes[i], \n                                        color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_channels[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:46.175112Z","iopub.execute_input":"2024-05-13T14:36:46.175456Z","iopub.status.idle":"2024-05-13T14:36:47.786294Z","shell.execute_reply.started":"2024-05-13T14:36:46.175427Z","shell.execute_reply":"2024-05-13T14:36:47.784955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n\n- Organic Search brings about the highest number of users & pageviews, but\n- Referral brings in close to half of the revenues.\n- Direct ranks the 3rd both in number of users & pageviews, yet second only to   \n    Referral in terms of generating revenues as users know pretty much what they want.\n- Social Media is not as effective in generating revenues as in generating the traffics.\n- Paid Search has got what the advertisers have paid for: equal amount of traffics, pageviews   \n    & revenues\n- Affiliates are not that much effective compared with other channels above,","metadata":{}},{"cell_type":"markdown","source":"### 3.2 'device_browser'","metadata":{}},{"cell_type":"code","source":"df_browser = df_train.groupby('device_browser').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False).head(10)\n\ndf_browser","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:47.787684Z","iopub.execute_input":"2024-05-13T14:36:47.788051Z","iopub.status.idle":"2024-05-13T14:36:48.018336Z","shell.execute_reply.started":"2024-05-13T14:36:47.788020Z","shell.execute_reply":"2024-05-13T14:36:48.017117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_browser.columns):\n    df_browser[col].sort_values().plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_browser[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('Browsers')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:48.019700Z","iopub.execute_input":"2024-05-13T14:36:48.020799Z","iopub.status.idle":"2024-05-13T14:36:49.457118Z","shell.execute_reply.started":"2024-05-13T14:36:48.020751Z","shell.execute_reply":"2024-05-13T14:36:49.455946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n   \n- Not suprising, Chrome is the powerhouse in all 3 fields, especially in    \n    generating the revenues, close to 90 percent of the market share.\n- Surprisingly, both Safari & Firefox outperforms Internet Explorer & Edge    \n    in all 3 fields.\n","metadata":{}},{"cell_type":"markdown","source":"### 3.3 'device_operatingSystem'","metadata":{}},{"cell_type":"code","source":"df_os = df_train.groupby('device_operatingSystem').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False).head(10)\n\ndf_os","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:49.458741Z","iopub.execute_input":"2024-05-13T14:36:49.459438Z","iopub.status.idle":"2024-05-13T14:36:49.690969Z","shell.execute_reply.started":"2024-05-13T14:36:49.459399Z","shell.execute_reply":"2024-05-13T14:36:49.689780Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_os.columns):\n    df_os[col].sort_values().plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_os[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('Operating Systems')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:49.692733Z","iopub.execute_input":"2024-05-13T14:36:49.693558Z","iopub.status.idle":"2024-05-13T14:36:51.081081Z","shell.execute_reply.started":"2024-05-13T14:36:49.693517Z","shell.execute_reply":"2024-05-13T14:36:51.079906Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.4 'device_isMobile'","metadata":{}},{"cell_type":"code","source":"df_mobile = df_train.groupby('device_isMobile').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False)\n\ndf_mobile","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:51.082562Z","iopub.execute_input":"2024-05-13T14:36:51.083300Z","iopub.status.idle":"2024-05-13T14:36:51.240765Z","shell.execute_reply.started":"2024-05-13T14:36:51.083262Z","shell.execute_reply":"2024-05-13T14:36:51.239461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Flatten the 2D array of axes\naxes = axes.flatten()\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_mobile.columns):\n    df_mobile[col].sort_values().plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_mobile[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('Mobile vs Non-Mobile')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:51.242466Z","iopub.execute_input":"2024-05-13T14:36:51.242982Z","iopub.status.idle":"2024-05-13T14:36:52.334713Z","shell.execute_reply.started":"2024-05-13T14:36:51.242938Z","shell.execute_reply":"2024-05-13T14:36:52.333816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 3.5 'device.deviceCategory'","metadata":{}},{"cell_type":"code","source":"df_category = df_train.groupby('device_deviceCategory').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False).head(10)\n\ndf_category","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:52.336040Z","iopub.execute_input":"2024-05-13T14:36:52.336959Z","iopub.status.idle":"2024-05-13T14:36:52.561801Z","shell.execute_reply.started":"2024-05-13T14:36:52.336926Z","shell.execute_reply":"2024-05-13T14:36:52.560946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_category.columns):\n    df_category[col].sort_values().plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_category[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('Device Category')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:52.563414Z","iopub.execute_input":"2024-05-13T14:36:52.564059Z","iopub.status.idle":"2024-05-13T14:36:53.681471Z","shell.execute_reply.started":"2024-05-13T14:36:52.564028Z","shell.execute_reply":"2024-05-13T14:36:53.680193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n\ndesktop & non-mobile devices\n- still dominate in bring in traffics, pageviews & revenues","metadata":{}},{"cell_type":"code","source":"device_cols = [\"device_browser\", \"device_deviceCategory\", \"device_operatingSystem\"]\n\ncolors = [\"#d6a5ff\", \"#fca6da\", \"#f4d39c\", \"#a9fcca\"]\ntraces = []\nfor i, col in enumerate(device_cols):\n    t = train[col].value_counts()\n    traces.append(go.Bar(marker=dict(color=colors[i]),orientation=\"h\", y = t.index[:15][::-1], x = t.values[:15][::-1]))\n\nfig = tools.make_subplots(rows=1, cols=3, subplot_titles=[\"Visits: Category\", \"Visits: Browser\",\"Visits: OS\"], print_grid=False)\nfig.append_trace(traces[1], 1, 1)\nfig.append_trace(traces[0], 1, 2)\nfig.append_trace(traces[2], 1, 3)\n\nfig['layout'].update(height=400, showlegend=False, title=\"Visits by Device Attributes\")\niplot(fig)\n\n## convert transaction revenue to float\ntrain[\"totals_transactionRevenue\"] = train[\"totals_transactionRevenue\"].astype('float')\n\ndevice_cols = [\"device_browser\", \"device_deviceCategory\", \"device_operatingSystem\"]\n\nfig = tools.make_subplots(rows=1, cols=3, subplot_titles=[\"Mean Revenue: Category\", \"Mean Revenue: Browser\",\"Mean Revenue: OS\"], print_grid=False)\n\ncolors = [\"red\", \"green\", \"purple\"]\ntrs = []\nfor i, col in enumerate(device_cols):\n    tmp = train.groupby(col).agg({\"totals_transactionRevenue\": \"mean\"}).reset_index().rename(columns={\"totals_transactionRevenue\" : \"Mean Revenue\"})\n    tmp = tmp.dropna().sort_values(\"Mean Revenue\", ascending = False)\n    tr = go.Bar(x = tmp[\"Mean Revenue\"][::-1], orientation=\"h\", marker=dict(opacity=0.5, color=colors[i]), y = tmp[col][::-1])\n    trs.append(tr)\n\nfig.append_trace(trs[1], 1, 1)\nfig.append_trace(trs[0], 1, 2)\nfig.append_trace(trs[2], 1, 3)\nfig['layout'].update(height=400, showlegend=False, title=\"Mean Revenue by Device Attributes\")\niplot(fig)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:53.683603Z","iopub.execute_input":"2024-05-13T14:36:53.684178Z","iopub.status.idle":"2024-05-13T14:36:56.584265Z","shell.execute_reply.started":"2024-05-13T14:36:53.684136Z","shell.execute_reply":"2024-05-13T14:36:56.582844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"###  GeoNetwork Attributes","metadata":{}},{"cell_type":"code","source":"geo_cols = ['geoNetwork_city', 'geoNetwork_continent','geoNetwork_country',\n            'geoNetwork_metro', 'geoNetwork_networkDomain', 'geoNetwork_region','geoNetwork_subContinent']\ngeo_cols = ['geoNetwork_continent','geoNetwork_subContinent']\n\ncolors = [\"#d6a5ff\", \"#fca6da\"]\nfig = tools.make_subplots(rows=1, cols=2, subplot_titles=[\"Visits : GeoNetwork Continent\", \"Visits : GeoNetwork subContinent\"], print_grid=False)\ntrs = []\nfor i,col in enumerate(geo_cols):\n    t = train[col].value_counts()\n    tr = go.Bar(x = t.index[:20], marker=dict(color=colors[i]), y = t.values[:20])\n    trs.append(tr)\n\nfig.append_trace(trs[0], 1, 1)\nfig.append_trace(trs[1], 1, 2)\nfig['layout'].update(height=400, margin=dict(b=150), showlegend=False)\niplot(fig)\n\n\n\n\ngeo_cols = ['geoNetwork_continent','geoNetwork_subContinent']\nfig = tools.make_subplots(rows=1, cols=2, subplot_titles=[\"Mean Revenue: Continent\", \"Mean Revenue: SubContinent\"], print_grid=False)\n\ncolors = [\"blue\", \"orange\"]\ntrs = []\nfor i, col in enumerate(geo_cols):\n    tmp = train.groupby(col).agg({\"totals_transactionRevenue\": \"mean\"}).reset_index().rename(columns={\"totals_transactionRevenue\" : \"Mean Revenue\"})\n    tmp = tmp.dropna().sort_values(\"Mean Revenue\", ascending = False)\n    tr = go.Bar(y = tmp[\"Mean Revenue\"], orientation=\"v\", marker=dict(opacity=0.5, color=colors[i]), x= tmp[col])\n    trs.append(tr)\n\nfig.append_trace(trs[0], 1, 1)\nfig.append_trace(trs[1], 1, 2)\nfig['layout'].update(height=450, margin=dict(b=200), showlegend=False)\niplot(fig)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:56.585882Z","iopub.execute_input":"2024-05-13T14:36:56.586331Z","iopub.status.idle":"2024-05-13T14:36:57.253104Z","shell.execute_reply.started":"2024-05-13T14:36:56.586293Z","shell.execute_reply":"2024-05-13T14:36:57.251700Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 4. 'geoNetwork' columns","metadata":{}},{"cell_type":"code","source":"geo_columns = ['geoNetwork_continent', 'geoNetwork_subContinent', 'geoNetwork_country', \n               'geoNetwork_region', 'geoNetwork_metro', 'geoNetwork_city', 'geoNetwork_networkDomain']\n\nfig, ax = plt.subplots(figsize=(20, 8))\n\nfor i, col in enumerate(geo_columns, 1):\n    plt.subplot(2, 4, i)\n    \n    ax = sns.countplot(y=train[col], order=train[col].value_counts()[:10].index,  )\n    plt.title(f'Count of {col}')    \n    \n    total = len(train[col])\n\n    for c in ax.containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        ax.bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.tight_layout()\nplt.show()\n\n# Explicitly close the figure to remove overlapping axes\nplt.close(fig)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:36:57.255100Z","iopub.execute_input":"2024-05-13T14:36:57.255569Z","iopub.status.idle":"2024-05-13T14:37:06.181398Z","shell.execute_reply.started":"2024-05-13T14:36:57.255528Z","shell.execute_reply":"2024-05-13T14:37:06.180183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.1 Global Analysis","metadata":{}},{"cell_type":"code","source":"df_country = df_train.groupby('geoNetwork_country').agg(aggregations\n                    ).sort_values(by=['totals_transactionRevenue'], \n                                  ascending=False).reset_index()\ndf_country.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:06.183331Z","iopub.execute_input":"2024-05-13T14:37:06.183686Z","iopub.status.idle":"2024-05-13T14:37:06.421424Z","shell.execute_reply.started":"2024-05-13T14:37:06.183655Z","shell.execute_reply":"2024-05-13T14:37:06.420233Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create basic choropleth map\nfig = px.choropleth(df_country, locationmode='country names', locations='geoNetwork_country', \n                    color=df_country['totals_transactionRevenue'], color_continuous_scale='Mint', \n                    hover_name='geoNetwork_country',  hover_data={'totals_transactionRevenue':':,.0f'},\n                    title='Revenue by Country')\n\nfig.update_layout(width=800, height=400, autosize=False, \n                  margin=dict(l=30, r=30, t=50, b=50, pad=10))\n\nfig.show(renderer='notebook')\n# fig.write_html(\"interactive_map.html\")","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:06.423399Z","iopub.execute_input":"2024-05-13T14:37:06.423851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 4.2 US Analysis","metadata":{}},{"cell_type":"code","source":"df_us = df_train[df_train['geoNetwork_country'] == 'United States'].reset_index(drop=True)\ndf_us","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:06.864067Z","iopub.execute_input":"2024-05-13T14:37:06.864466Z","iopub.status.idle":"2024-05-13T14:37:07.204188Z","shell.execute_reply.started":"2024-05-13T14:37:06.864434Z","shell.execute_reply":"2024-05-13T14:37:07.203107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add a column with state abbreviations\nstate_abbreviations = {'Alabama': 'AL', 'Alaska': 'AK', 'Arizona': 'AZ', 'Arkansas': 'AR', 'California': 'CA',\n                       'Colorado': 'CO', 'Connecticut': 'CT', 'Delaware': 'DE', 'Florida': 'FL', 'Georgia': 'GA',\n                       'Hawaii': 'HI', 'Idaho': 'ID', 'Illinois': 'IL', 'Indiana': 'IN', 'Iowa': 'IA',\n                       'Kansas': 'KS', 'Kentucky': 'KY', 'Louisiana': 'LA', 'Maine': 'ME', 'Maryland': 'MD',\n                       'Massachusetts': 'MA', 'Michigan': 'MI', 'Minnesota': 'MN', 'Mississippi': 'MS',\n                       'Missouri': 'MO', 'Montana': 'MT', 'Nebraska': 'NE', 'Nevada': 'NV', 'New Hampshire': 'NH',\n                       'New Jersey': 'NJ', 'New Mexico': 'NM', 'New York': 'NY', 'North Carolina': 'NC',\n                       'North Dakota': 'ND', 'Ohio': 'OH', 'Oklahoma': 'OK', 'Oregon': 'OR', 'Pennsylvania': 'PA',\n                       'Rhode Island': 'RI', 'South Carolina': 'SC', 'South Dakota': 'SD', 'Tennessee': 'TN',\n                       'Texas': 'TX', 'Utah': 'UT', 'Vermont': 'VT', 'Virginia': 'VA', 'Washington': 'WA',\n                       'West Virginia': 'WV', 'Wisconsin': 'WI', 'Wyoming': 'WY', 'District of Columbia': 'DC' }\n\ndf_us['stateCode'] = df_us['geoNetwork_region'].map(state_abbreviations)\ndf_us.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:07.205927Z","iopub.execute_input":"2024-05-13T14:37:07.206280Z","iopub.status.idle":"2024-05-13T14:37:07.273155Z","shell.execute_reply.started":"2024-05-13T14:37:07.206252Z","shell.execute_reply":"2024-05-13T14:37:07.272078Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_states = df_us.groupby(['geoNetwork_region', 'stateCode']).agg(aggregations).sort_values(\n                                            by='totals_transactionRevenue', ascending=False).reset_index()\ndf_states.head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:07.274488Z","iopub.execute_input":"2024-05-13T14:37:07.274822Z","iopub.status.idle":"2024-05-13T14:37:07.453812Z","shell.execute_reply.started":"2024-05-13T14:37:07.274794Z","shell.execute_reply":"2024-05-13T14:37:07.452562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create basic choropleth map\nfig = px.choropleth(df_states, locationmode='USA-states', locations='stateCode',\n                    color=df_states['totals_transactionRevenue'], hover_name='geoNetwork_region', \n                    hover_data={'totals_transactionRevenue':':,.0f'}, \n                    title='Revenue by State', color_continuous_scale='Mint')\n                  \nfig.update_layout(width=800, height=400, autosize=False, geo_scope='usa',\n                  margin=dict(l=30, r=30, t=50, b=50, pad=10))\n\n# For interactive display in a Jupyter Notebook:\nfig.show(renderer='notebook')\n# fig.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:07.455406Z","iopub.execute_input":"2024-05-13T14:37:07.456079Z","iopub.status.idle":"2024-05-13T14:37:07.609637Z","shell.execute_reply.started":"2024-05-13T14:37:07.456048Z","shell.execute_reply":"2024-05-13T14:37:07.608202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 6. 'trafficSource' Columns","metadata":{}},{"cell_type":"code","source":"traffic_columns = ['trafficSource_campaign', 'trafficSource_source', 'trafficSource_medium', \n                   'trafficSource_keyword', 'trafficSource_referralPath', \n                   'trafficSource_adwordsClickInfo.page', 'trafficSource_adwordsClickInfo.slot', \n                   'trafficSource_adwordsClickInfo.adNetworkType', 'trafficSource_adContent']\n\nfig, ax = plt.subplots(figsize=(18, 12))\n\nfor i, col in enumerate(traffic_columns, 1):\n    plt.subplot(3, 3, i)\n    \n    # Exclude 'others' & '(not set)' from the specified column\n    filtered_df = train[(train[col] != 'others') & \n                        (train[col] != '(not provided)') & \n                        (train[col] != '(not set)')]\n    \n    ax = sns.countplot(y=filtered_df[col], order=filtered_df[col].value_counts()[:7].index)\n    plt.title(f'Count of {col}')\n\n    total = len(train[col])\n\n    for c in ax.containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        ax.bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.tight_layout()\nplt.show()\n\n# Explicitly close the figure to remove overlapping axes\nplt.close(fig)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:07.611469Z","iopub.execute_input":"2024-05-13T14:37:07.612002Z","iopub.status.idle":"2024-05-13T14:37:20.088341Z","shell.execute_reply.started":"2024-05-13T14:37:07.611964Z","shell.execute_reply":"2024-05-13T14:37:20.086970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"In this section, we have excluded 'others' & '(not set)' from the specified columns for they comprise    \nan overwhelming large part of the data in this section.\n\nWe are only interested in the foloowing features:   \n\n- 'trafficSource.source'    \n- 'trafficSource.medium'   ","metadata":{}},{"cell_type":"markdown","source":"### 6.1 'trafficSource.source'\n\n- The source of the traffic source. Could be the name of the search engine, etc.\n- Top 10 sources are:   \n    'google', 'youtube.com', '(direct)', 'mall.googleplex.com', 'Partners',   \n    'analytics.google.com', 'dfa', 'google.com', 'm.facebook.com', 'baidu'.      \n    google: 44% of the trafficsource source","metadata":{}},{"cell_type":"code","source":"df_source = df_train.groupby('trafficSource_source').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False).head(10)\n\ndf_source","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:20.090182Z","iopub.execute_input":"2024-05-13T14:37:20.090553Z","iopub.status.idle":"2024-05-13T14:37:20.324864Z","shell.execute_reply.started":"2024-05-13T14:37:20.090520Z","shell.execute_reply":"2024-05-13T14:37:20.324023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_source.columns):\n    df_source[col].sort_values().plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_source[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('TrafficSource Source')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:20.326249Z","iopub.execute_input":"2024-05-13T14:37:20.326574Z","iopub.status.idle":"2024-05-13T14:37:21.706156Z","shell.execute_reply.started":"2024-05-13T14:37:20.326546Z","shell.execute_reply":"2024-05-13T14:37:21.705036Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n\n'trafficSource_source'    \n    \n- 'google' brings in most of the users & pageviews, while 'mall.googleplex.com' generates most of the revenues.\n- '(direct)' ranks second in all 3 fields. No wonder again, users know what they want & type the urls in the browser.\n- 'dfa': DFA source refers to DoubleClick For Advertisers, which is the old brand name for Google Marketing Platform. ","metadata":{}},{"cell_type":"markdown","source":"### 6.2 'trafficSource_medium'\n\n- The medium of the traffic source. \"organic\", \"cpc\", \"referral\", etc,   \n- Top 6 sources are:   \n    'organic', 'referral', '(none)', 'cpc', 'affiliate', 'cpm', etc.   \n    organic: 42% of the traffucesource medium","metadata":{}},{"cell_type":"code","source":"df_medium = df_train.groupby('trafficSource_medium').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False).head(10)\n\ndf_medium","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:21.707681Z","iopub.execute_input":"2024-05-13T14:37:21.708028Z","iopub.status.idle":"2024-05-13T14:37:21.937692Z","shell.execute_reply.started":"2024-05-13T14:37:21.707998Z","shell.execute_reply":"2024-05-13T14:37:21.936172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_medium.columns):\n    df_medium[col].sort_values().plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_medium[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('TrafficSource Medium')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:21.939783Z","iopub.execute_input":"2024-05-13T14:37:21.940267Z","iopub.status.idle":"2024-05-13T14:37:23.236077Z","shell.execute_reply.started":"2024-05-13T14:37:21.940227Z","shell.execute_reply":"2024-05-13T14:37:23.234925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n\n- \"organic\" bring about most of the sessions & pageviews, it is   \n    \"referral\" that generates most of the revenues. \n- \"cpc\": Cost Per Click. Plays an important role in all 3 fields.\n- '(none)': means Analytics doesn't have any referral information for these users.","metadata":{}},{"cell_type":"markdown","source":"#### 6.3 Source & Medium","metadata":{}},{"cell_type":"code","source":"df_source_medium = df_train.groupby(['trafficSource_source', 'trafficSource_medium']).agg(aggregations)\n\n# df_source_medium.sort_values(by='revenue', ascending=False).head(10)\ndf_source_medium","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:23.237592Z","iopub.execute_input":"2024-05-13T14:37:23.237951Z","iopub.status.idle":"2024-05-13T14:37:23.565132Z","shell.execute_reply.started":"2024-05-13T14:37:23.237923Z","shell.execute_reply":"2024-05-13T14:37:23.563788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Create subplots\nfig, axes = plt.subplots(nrows=1, ncols=3, figsize=(15, 5))\n# Flatten the 2D array of axes\n# axes = axes.flatten()\n\n# Plot each column in a separate subplot\nfor i, col in enumerate(df_source_medium.columns):\n    df_source_medium[col].sort_values(ascending=False)[:10].plot(kind='barh', ax=axes[i], color=colors)\n    axes[i].set_title(col)\n    axes[i].set_xlabel('Count')\n\n    total = df_source_medium[col].sum()\n\n    for c in axes[i].containers:\n        percentages = [f'{(v / total) * 100:.2f}%' for v in c.datavalues]\n        axes[i].bar_label(c, labels=percentages, fmt='%s', padding=1)\n\nplt.suptitle('TrafficSource Source & Medium')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:23.566672Z","iopub.execute_input":"2024-05-13T14:37:23.567053Z","iopub.status.idle":"2024-05-13T14:37:24.987121Z","shell.execute_reply.started":"2024-05-13T14:37:23.567022Z","shell.execute_reply":"2024-05-13T14:37:24.985922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n\n- 'google organic' generates most of the traffics, the rererrals from    \n    'mall.googleplex.com' generates most of the revenue.\n- 'direct none': again, Analytics does not have any information about users,   \n    yet are dominant in all 3 fields on average after 'google organic'.","metadata":{}},{"cell_type":"markdown","source":"## 7. Revenue = 0 vs Revenue > 0","metadata":{}},{"cell_type":"code","source":"# revenue = 0 transactions\ndf_revenue_zero = df_train[df_train['totals_transactionRevenue'] == 0]\ndf_revenue_zero.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:24.988660Z","iopub.execute_input":"2024-05-13T14:37:24.989904Z","iopub.status.idle":"2024-05-13T14:37:25.207040Z","shell.execute_reply.started":"2024-05-13T14:37:24.989844Z","shell.execute_reply":"2024-05-13T14:37:25.205883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# revenue > 0 transactions\ndf_revenue_nonzero = df_train[df_train['totals_transactionRevenue'] > 0].reset_index(drop=True)\ndf_revenue_nonzero.head()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:25.208502Z","iopub.execute_input":"2024-05-13T14:37:25.209451Z","iopub.status.idle":"2024-05-13T14:37:25.243678Z","shell.execute_reply.started":"2024-05-13T14:37:25.209418Z","shell.execute_reply":"2024-05-13T14:37:25.242290Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"non_zero_transact_pct = len(df_revenue_nonzero) / len(df_train) * 100\nprint(f'Non-Zero Transaction Percentage: {non_zero_transact_pct:.02f}%')\n\nzero_transact_pct = len(df_revenue_zero) / len(df_train) * 100\nprint(f'Zero Transaction Percentage: {zero_transact_pct:.02f}%')","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:25.245639Z","iopub.execute_input":"2024-05-13T14:37:25.246396Z","iopub.status.idle":"2024-05-13T14:37:25.253414Z","shell.execute_reply.started":"2024-05-13T14:37:25.246360Z","shell.execute_reply":"2024-05-13T14:37:25.252053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">***Observations:***\n\n- 98.73% of the traffics (sessions, pageviews) does not generate revenues at all.","metadata":{}},{"cell_type":"code","source":"# Filter the DataFrame for zero revenue\n# df_revenue_zero = df_train[df_train['totals.transactionRevenue'] == 0]\n\n# List of columns to loop through\ncolumns_to_loop = ['channelGrouping', 'trafficSource_source', 'trafficSource_medium']\n\n# Create subplots\nfig, axs = plt.subplots(1, 3, figsize=(15, 5))\n\n# Loop through each column and subplot\nfor i, column in enumerate(columns_to_loop):\n    # Create the aggregated DataFrame\n    df_column_revenue_zero = df_revenue_zero.groupby(column).agg(\n        count=(column, 'count') ).sort_values(by='count', ascending=False).head(10)\n\n    # Calculate the percentage\n    total_count = df_column_revenue_zero['count'].sum()\n    df_column_revenue_zero['percentage'] = (df_column_revenue_zero['count'] / total_count) * 100\n\n    # Plot the horizontal bar chart in the current subplot\n    bar_chart = df_column_revenue_zero['count'].plot(kind='barh', ax=axs[i], color=colors)\n\n    # Add labels and title\n    axs[i].set_xlabel(f'Count of Zero Transactions ({column})')\n    axs[i].set_ylabel(column)\n    axs[i].set_title(f'Top 10 {column} with Zero Transactions')\n\n    # Display percentages on the right side of the bars\n    for j, patch in enumerate(bar_chart.patches):\n        width = patch.get_width()\n        percentage = df_column_revenue_zero['percentage'].iloc[j]\n        axs[i].text(width + 1000, patch.get_y() + patch.get_height() / 2,\n                    f'{percentage:.2f}%', ha='left', va='center', fontsize=10)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:25.255722Z","iopub.execute_input":"2024-05-13T14:37:25.256281Z","iopub.status.idle":"2024-05-13T14:37:27.307895Z","shell.execute_reply.started":"2024-05-13T14:37:25.256241Z","shell.execute_reply":"2024-05-13T14:37:27.306677Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter the DataFrame for non-zero revenue\n# df_revenue_nonzero = df_train[df_train['totals_transactionRevenue'] > 0]\n\n# List of columns to loop through\ncolumns_to_loop = ['channelGrouping', 'trafficSource_source', 'trafficSource_medium']\n\n# Plot the bar chart\nfig, ax = plt.subplots(3, 1, figsize=(10, 9))\n\n# Loop through each column\nfor i, col in enumerate(columns_to_loop):\n    # Create the aggregated DataFrame\n    df_column_revenue_nonzero = df_revenue_nonzero.groupby(col).agg(\n        count=(col, 'count')\n    ).sort_values(by='count', ascending=False).head(10)\n\n    # Calculate the percentage\n    total_count = df_column_revenue_nonzero['count'].sum()\n    df_column_revenue_nonzero['percentage'] = (df_column_revenue_nonzero['count'] / total_count) * 100\n\n    bar_chart = df_column_revenue_nonzero['count'].plot(kind='bar', rot=0, ax=ax[i], color=colors)\n\n    # Add labels and title\n    ax[i].set_ylabel(f'Count of Non-Zero Transactions ({col})')\n    ax[i].set_xlabel(col)\n    ax[i].set_title(f'Top 10 {col} with Non-Zero Transactions')\n\n    # Display percentages on top of the bars\n    for j, patch in enumerate(bar_chart.patches):\n        height = patch.get_height()\n        percentage = df_column_revenue_nonzero['percentage'].iloc[j]\n        ax[i].text(patch.get_x() + patch.get_width() / 2,\n                height + 1000, f'{percentage:.2f}%', ha='center', va='bottom', fontsize=10)\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:27.309828Z","iopub.execute_input":"2024-05-13T14:37:27.310285Z","iopub.status.idle":"2024-05-13T14:37:28.659624Z","shell.execute_reply.started":"2024-05-13T14:37:27.310248Z","shell.execute_reply":"2024-05-13T14:37:28.658254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Filter the DataFrame for zero revenue\n# df_revenue_zero = df_train[df_train['totals_transactionRevenue'] == 0]\n\n# List of columns to loop through\ncolumns_to_loop = ['channelGrouping', 'trafficSource_source', 'trafficSource_medium']\n\n# Loop through each column\nfor col in columns_to_loop:\n    # Create the aggregated DataFrame\n    df_column_revenue_zero = df_revenue_zero.groupby(col).agg(\n        count=(col, 'count')\n    ).sort_values(by='count', ascending=False).head(10)\n\n    # Calculate the percentage\n    total_count = df_column_revenue_zero['count'].sum()\n    df_column_revenue_zero['percentage'] = (df_column_revenue_zero['count'] / total_count) * 100\n\n    # Plot the bar chart\n    fig, ax = plt.subplots(figsize=(12, 4))\n    # Flatten the 2D array of axes\n    # axes = axes.flatten()\n\n    bar_chart = df_column_revenue_zero['count'].plot(kind='bar', rot=0, ax=ax, color=colors)\n\n    # Add labels and title\n    ax.set_ylabel(f'Count of Zero Transactions ({col})')\n    ax.set_xlabel(col)\n    ax.set_title(f'Top 10 {col} with Zero Transactions')\n\n    # Display percentages on top of the bars\n    for i, patch in enumerate(bar_chart.patches):\n        height = patch.get_height()\n        percentage = df_column_revenue_zero['percentage'].iloc[i]\n        ax.text(patch.get_x() + patch.get_width() / 2,\n                height + 1000, f'{percentage:.2f}%', ha='center', va='bottom', fontsize=12)\n\n    plt.tight_layout()\n    plt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:28.661503Z","iopub.execute_input":"2024-05-13T14:37:28.662122Z","iopub.status.idle":"2024-05-13T14:37:30.671261Z","shell.execute_reply.started":"2024-05-13T14:37:28.662086Z","shell.execute_reply":"2024-05-13T14:37:30.669943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":">**Observations:**\n\n- Most of the zero-revenue transacations are from 'organic search' or 'search google'\n- while most of the non-zero transactions are from 'mall.googleplex.com' & 'referral',    \n    '(direct)' plays an important role in bringing about traffics & revenues.","metadata":{}},{"cell_type":"markdown","source":"## 8. User & Session Analysis","metadata":{}},{"cell_type":"markdown","source":"### 8.1 User Analysis","metadata":{}},{"cell_type":"code","source":"df_visitors = df_train.groupby('fullVisitorId').agg(aggregations).sort_values(\n                    by='totals_transactionRevenue', ascending=False )\n\ndf_visitors.drop('fullVisitorId', axis=1).head(10)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:30.673247Z","iopub.execute_input":"2024-05-13T14:37:30.674417Z","iopub.status.idle":"2024-05-13T14:37:33.982288Z","shell.execute_reply.started":"2024-05-13T14:37:30.674375Z","shell.execute_reply":"2024-05-13T14:37:33.981003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Users with non-zero revenue transactions\ndf_users_revenue = df_revenue_nonzero.groupby('fullVisitorId').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False)\n\ndf_users_revenue.drop('fullVisitorId', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:33.983976Z","iopub.execute_input":"2024-05-13T14:37:33.984318Z","iopub.status.idle":"2024-05-13T14:37:34.026036Z","shell.execute_reply.started":"2024-05-13T14:37:33.984289Z","shell.execute_reply":"2024-05-13T14:37:34.024776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Distribution of non-zero revenue transactions\nnon_zero = df_train[df_train['totals_transactionRevenue'] > 0] \\\n                 ['totals_transactionRevenue'].reset_index(drop=True)\n    \nfig, ax = plt.subplots(figsize=(10, 5))\n\nsns.histplot(non_zero, kde=True, bins=30, log_scale=True)\nticks_x = ticker.FuncFormatter(lambda x, pos: '{:,.0f}'.format(x/1000))\nax.xaxis.set_major_formatter(ticks_x)\n\nplt.title('Distribution of Non-Zero Revenue Transactions')\nplt.xlabel('Transaction Revenue in Thousands')\n\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:34.027616Z","iopub.execute_input":"2024-05-13T14:37:34.028081Z","iopub.status.idle":"2024-05-13T14:37:35.437460Z","shell.execute_reply.started":"2024-05-13T14:37:34.028041Z","shell.execute_reply":"2024-05-13T14:37:35.436358Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Users with zero revenue transactions\ndf_user_revenue_zero = df_revenue_zero.groupby('fullVisitorId').agg(aggregations\n                ).sort_values(by='totals_transactionRevenue', ascending=False)\n\ndf_user_revenue_zero.drop('fullVisitorId', axis=1)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:35.439290Z","iopub.execute_input":"2024-05-13T14:37:35.440065Z","iopub.status.idle":"2024-05-13T14:37:38.602267Z","shell.execute_reply.started":"2024-05-13T14:37:35.440022Z","shell.execute_reply":"2024-05-13T14:37:38.601341Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_user_revenue_zero_pct = round(len(df_user_revenue_zero['totals_transactionRevenue']) / \n                                 len(df_train['totals_transactionRevenue']) * 100, 2)\nprint(f'Percentage of users witn zero-revenue trasactions:\\n {df_user_revenue_zero_pct}%')","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:38.603440Z","iopub.execute_input":"2024-05-13T14:37:38.603955Z","iopub.status.idle":"2024-05-13T14:37:38.610008Z","shell.execute_reply.started":"2024-05-13T14:37:38.603926Z","shell.execute_reply":"2024-05-13T14:37:38.608922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 8.2 Sessions","metadata":{}},{"cell_type":"code","source":"# Non-zero revenue sessions\ndf_session_revenue = df_revenue_nonzero.groupby('sessionId').agg(\n                count=('sessionId', 'count'),\n                pageviews=('totals_pageviews', 'sum'),\n                revenue=('totals_transactionRevenue', 'sum')\n                ).sort_values(by='revenue', ascending=False)\n\ndf_session_revenue","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:38.611821Z","iopub.execute_input":"2024-05-13T14:37:38.612301Z","iopub.status.idle":"2024-05-13T14:37:38.667017Z","shell.execute_reply.started":"2024-05-13T14:37:38.612232Z","shell.execute_reply":"2024-05-13T14:37:38.665949Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Sessions with zero revenue transactions\ndf_session_revenue_zero = df_revenue_zero.groupby('sessionId').agg(\n                sessions=('sessionId', 'count'),\n                pageviews=('totals_pageviews', 'sum'),\n                revenue=('totals_transactionRevenue', 'sum')\n                ).sort_values(by='revenue', ascending=False)\n\ndf_session_revenue_zero","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:38.668226Z","iopub.execute_input":"2024-05-13T14:37:38.668804Z","iopub.status.idle":"2024-05-13T14:37:42.662871Z","shell.execute_reply.started":"2024-05-13T14:37:38.668774Z","shell.execute_reply":"2024-05-13T14:37:42.661405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_session_revenue_zero_pct = round(len(df_session_revenue_zero['revenue']) / \n                                    len(df_train['totals_transactionRevenue']) * 100, 2)\n\nprint(f'Percentage of sessions witn zero-revenue trasactions:\\n {df_session_revenue_zero_pct}%')","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:42.664526Z","iopub.execute_input":"2024-05-13T14:37:42.665007Z","iopub.status.idle":"2024-05-13T14:37:42.673650Z","shell.execute_reply.started":"2024-05-13T14:37:42.664970Z","shell.execute_reply":"2024-05-13T14:37:42.672372Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 9. Time Analysis","metadata":{}},{"cell_type":"markdown","source":"### 9.1 Extract 'Year', 'Month', 'Month_Num' & 'Day' from 'date' column","metadata":{}},{"cell_type":"code","source":"# Extract 'Year', 'Month', 'Month_Num' & 'Day' from 'date'\ndef create_features(data):\n    \"\"\"\n    Create time series features based on time series index.\n    \"\"\"\n\n    # data = data.copy()\n    \n    # Convert 'date' column to datetime series \n    data['date'] = pd.to_datetime(data['date'], format='%Y%m%d')\n\n    # Extract 'Year', 'Month', 'Month_Num' & 'Day' from 'Order Date'\n    data['Year'] = data.date.dt.year\n    data['Month'] = data.date.dt.strftime('%b') # abbrev month name\n    data['Month_Num'] = data.date.dt.month\n    data['Day'] = data.date.dt.day\n\n    # Extract 'Weekday' & 'Week Day Num'\n    data['Weekday'] = data['date'].dt.strftime('%a')\n    data['Weekday_Num'] = data.date.dt.day_of_week + 1 # Week start on Monday as 0, so + 1\n\n    return data","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:42.675201Z","iopub.execute_input":"2024-05-13T14:37:42.675784Z","iopub.status.idle":"2024-05-13T14:37:42.689124Z","shell.execute_reply.started":"2024-05-13T14:37:42.675752Z","shell.execute_reply":"2024-05-13T14:37:42.687866Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = df_train.copy()\n\ncreate_features(df_train)\n\ndf_train = df_train[df_train['date'] != '2017-08-01']\ndf_train.tail()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:42.690896Z","iopub.execute_input":"2024-05-13T14:37:42.691247Z","iopub.status.idle":"2024-05-13T14:37:57.972027Z","shell.execute_reply.started":"2024-05-13T14:37:42.691218Z","shell.execute_reply":"2024-05-13T14:37:57.970919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.2 Daily Transactions","metadata":{}},{"cell_type":"code","source":"import matplotlib.ticker as ticker","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:57.973540Z","iopub.execute_input":"2024-05-13T14:37:57.974004Z","iopub.status.idle":"2024-05-13T14:37:57.979529Z","shell.execute_reply.started":"2024-05-13T14:37:57.973973Z","shell.execute_reply":"2024-05-13T14:37:57.978183Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_daily = df_train.groupby('date').agg(aggregations)\ndf_daily","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:57.981177Z","iopub.execute_input":"2024-05-13T14:37:57.982073Z","iopub.status.idle":"2024-05-13T14:37:58.144621Z","shell.execute_reply.started":"2024-05-13T14:37:57.982042Z","shell.execute_reply":"2024-05-13T14:37:58.143369Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(3, 1, figsize=(15, 6), sharex=True)\n\nticks_y = ticker.FuncFormatter(lambda x, pos: '{:,.0f}'.format(x/1000000))\n\nsns.lineplot(data=df_daily, x='date', y='totals_transactionRevenue', color='g', label='Revenue in Millions', ax=ax[0])\n# plt.gca(ax=ax[0]).yaxis.set_major_formatter(ticks_y)\nax[0].yaxis.set_major_formatter(ticks_y)\n\n# sns.lineplot(data=df_daily, x='date', y='totals.hits', color='b', label='Hits', ax=ax[1])\nsns.lineplot(data=df_daily, x='date', y='totals_pageviews', color='b', label='Pageviews', ax=ax[1])\n\nsns.lineplot(data=df_daily, x='date', y='fullVisitorId', color='k', label='Visitors', ax=ax[2])\n\nplt.legend()\nplt.suptitle('Daily Visitors, Pageviews & Revenue')\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:58.147174Z","iopub.execute_input":"2024-05-13T14:37:58.147596Z","iopub.status.idle":"2024-05-13T14:37:59.743975Z","shell.execute_reply.started":"2024-05-13T14:37:58.147558Z","shell.execute_reply":"2024-05-13T14:37:59.742961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.3 Weekly Anslysis","metadata":{}},{"cell_type":"code","source":"df_week = df_train.groupby(['Weekday', 'Weekday_Num']).agg(aggregations)\n\ndf_week.sort_values(by='Weekday_Num', inplace=True)\ndf_week ","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:37:59.757659Z","iopub.execute_input":"2024-05-13T14:37:59.758613Z","iopub.status.idle":"2024-05-13T14:38:00.026966Z","shell.execute_reply.started":"2024-05-13T14:37:59.758575Z","shell.execute_reply":"2024-05-13T14:38:00.025916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(15, 4), sharex=False)\n\nsns.lineplot(data=df_week, x='Weekday', y='fullVisitorId', color='k', label='Visitors', ax=ax[0])\nsns.lineplot(data=df_week, x='Weekday', y='totals_pageviews', color='b', label='Pageviews', ax=ax[1])\n\nticks_y = ticker.FuncFormatter(lambda x, pos: '{:,.0f}'.format(x/1000000))\nsns.lineplot(data=df_week, x='Weekday', y='totals_transactionRevenue', color='g', label='Revenuev in Millions', ax=ax[2])\nax[2].yaxis.set_major_formatter(ticks_y)\n\nplt.legend()\nplt.suptitle('Weekly Visitors, Pageview & Revenue')\n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:00.028530Z","iopub.execute_input":"2024-05-13T14:38:00.028970Z","iopub.status.idle":"2024-05-13T14:38:01.032384Z","shell.execute_reply.started":"2024-05-13T14:38:00.028939Z","shell.execute_reply":"2024-05-13T14:38:01.031130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 9.4 Monthly Analysis","metadata":{}},{"cell_type":"code","source":"df_train_dates = df_train.set_index(df_train['date']).drop('date', axis=1)\n\n# months = df_train_dates['Month'].unique()\nmonths = ['Aug', 'Sep', 'Oct', 'Nov', 'Dec', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul']\n\n# Set up subplots\nfig, axes = plt.subplots(nrows=len(months), ncols=1, \n                         figsize=(10, 3*len(months)))\n\n# fig.suptitle('Monthly Revenues in Millions')\n\n# Plot for each year\nfor i, month in enumerate(months):\n    df_monthly = df_train_dates[df_train_dates['Month'] == month]\n    df_monthly_rev = df_monthly['totals_transactionRevenue']\n\n    # Set subplot title\n    axes[i].set_title(f'Monthly Revenues in Millions - {month}')\n\n    ticks_y = ticker.FuncFormatter(lambda x, pos: '{:,.0f}'.format(x/1000000))\n\n    # Plot on the i-th subplot\n    # df_month_rev.plot(kind='line', ax=axes[i], rot=0)\n    sns.lineplot(data=df_monthly_rev, ax=axes[i], legend=True)\n    axes[i].yaxis.set_major_formatter(ticks_y)\n\n    # Set y-axis scale\n    # axes[i].set_yscale('log') \n\nplt.tight_layout()\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:01.034359Z","iopub.execute_input":"2024-05-13T14:38:01.034780Z","iopub.status.idle":"2024-05-13T14:38:31.387379Z","shell.execute_reply.started":"2024-05-13T14:38:01.034740Z","shell.execute_reply":"2024-05-13T14:38:31.386241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_month = df_train.groupby(['Year', 'Month', 'Month_Num']).agg(aggregations)\ndf_month = df_month[df_month.index != '2017-08-01']\n\ndf_month.sort_values(by=['Year', 'Month_Num'], inplace=True)\ndf_month = df_month.reset_index()\n\ndf_month","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:31.388847Z","iopub.execute_input":"2024-05-13T14:38:31.389471Z","iopub.status.idle":"2024-05-13T14:38:31.691338Z","shell.execute_reply.started":"2024-05-13T14:38:31.389440Z","shell.execute_reply":"2024-05-13T14:38:31.689889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 10. Machine Learning Model","metadata":{}},{"cell_type":"code","source":"import lightgbm as lgb \nimport xgboost as xgb\nfrom catboost import CatBoostRegressor\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder, StandardScaler\nfrom sklearn.metrics import mean_squared_error","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:31.692850Z","iopub.execute_input":"2024-05-13T14:38:31.693297Z","iopub.status.idle":"2024-05-13T14:38:33.307788Z","shell.execute_reply.started":"2024-05-13T14:38:31.693259Z","shell.execute_reply":"2024-05-13T14:38:33.306524Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### 10.1 Data Transformation","metadata":{}},{"cell_type":"code","source":"### Create calendar columns in test dataset\ndf_test = df_test.copy()\n\ncreate_features(df_test)\ndf_test.tail()","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:33.309389Z","iopub.execute_input":"2024-05-13T14:38:33.310081Z","iopub.status.idle":"2024-05-13T14:38:46.709334Z","shell.execute_reply.started":"2024-05-13T14:38:33.310047Z","shell.execute_reply":"2024-05-13T14:38:46.707740Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_set = df_train.drop(columns=['Year', 'Month', 'Weekday'], axis=1).reset_index(drop=True)\ntest_set = df_test.drop(columns=['Year', 'Month', 'Weekday'], axis=1).reset_index(drop=True)\n\nprint(train_set.shape, test_set.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:46.711150Z","iopub.execute_input":"2024-05-13T14:38:46.711587Z","iopub.status.idle":"2024-05-13T14:38:47.354384Z","shell.execute_reply.started":"2024-05-13T14:38:46.711556Z","shell.execute_reply":"2024-05-13T14:38:47.353531Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# numeric_columns = train_set.select_dtypes(include='number').columns\nnumeric_cols = [c for c in train_set.columns if c.startswith('total') and c != 'totals_transactionRevenue']\nnumeric_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:47.355787Z","iopub.execute_input":"2024-05-13T14:38:47.356340Z","iopub.status.idle":"2024-05-13T14:38:47.364187Z","shell.execute_reply.started":"2024-05-13T14:38:47.356309Z","shell.execute_reply":"2024-05-13T14:38:47.363008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in numeric_cols:\n\n    # Check for and handle missing values\n    train_set[col].fillna(0, inplace=True) \n    test_set[col].fillna(0, inplace=True)\n\n    sc = StandardScaler()\n    # Fit the StandardScaler on the combined data (train and test)\n    sc.fit(pd.concat([train_set[[col]], test_set[[col]]], axis=0, ignore_index=True))\n\n    # Transform the values in both train and test sets\n    train_set[col] = sc.transform(train_set[[col]])\n    test_set[col] = sc.transform(test_set[[col]])","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:47.366292Z","iopub.execute_input":"2024-05-13T14:38:47.366751Z","iopub.status.idle":"2024-05-13T14:38:47.503455Z","shell.execute_reply.started":"2024-05-13T14:38:47.366702Z","shell.execute_reply":"2024-05-13T14:38:47.502046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# categorical_columns = train_set.select_dtypes(include='object').columns\n\n## exclude columns for normalization or label-encoding\ncols_excluded = ['date', 'visitNumber']\n\ncategorical_cols = [c for c in train_set.columns if c not in cols_excluded and not c.startswith('total')]\ncategorical_cols","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:47.505022Z","iopub.execute_input":"2024-05-13T14:38:47.505935Z","iopub.status.idle":"2024-05-13T14:38:47.517484Z","shell.execute_reply.started":"2024-05-13T14:38:47.505896Z","shell.execute_reply":"2024-05-13T14:38:47.515924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Convert date to UNIX timestamp\ntrain_set['date'] = train['date'].astype(int)\ntest_set['date'] = test['date'].astype(int)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:47.519246Z","iopub.execute_input":"2024-05-13T14:38:47.519743Z","iopub.status.idle":"2024-05-13T14:38:47.589583Z","shell.execute_reply.started":"2024-05-13T14:38:47.519695Z","shell.execute_reply":"2024-05-13T14:38:47.588502Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in categorical_cols:\n\n    le = LabelEncoder()\n    # Fit the LabelEncoder on the combined data (train and test)\n    le.fit(pd.concat([train_set[col], test_set[col]], axis=0, ignore_index=True))\n    \n    # Transform the values in both train and test sets\n    train_set[col] = le.transform(train_set[col])\n    test_set[col] = le.transform(test_set[col])","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:38:47.591280Z","iopub.execute_input":"2024-05-13T14:38:47.591617Z","iopub.status.idle":"2024-05-13T14:39:14.715238Z","shell.execute_reply.started":"2024-05-13T14:38:47.591589Z","shell.execute_reply":"2024-05-13T14:39:14.713956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = train_set.drop('totals_transactionRevenue', axis=1)\ny = train_set['totals_transactionRevenue']\n\nX_train, X_test, y_train, y_test = train_test_split( X, y, test_size=0.20, random_state=42)\nprint(X_train.shape, X_test.shape, y_train.shape, y_test.shape)","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:39:14.718982Z","iopub.execute_input":"2024-05-13T14:39:14.719981Z","iopub.status.idle":"2024-05-13T14:39:15.207013Z","shell.execute_reply.started":"2024-05-13T14:39:14.719931Z","shell.execute_reply":"2024-05-13T14:39:15.205410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"params = {'num_leaves': 38,              \n          'min_data_in_leaf': 50,\n          'objective': 'regression',     \n          'max_depth': -1,                \n          'learning_rate': 0.1,           \n          'verbose': -1,\n          'predict_disable_shape_check': [True]\n          }\n\ntrain_data = lgb.Dataset(X_train, y_train)\ntest_data = lgb.Dataset(X_test, y_test)\n\nmodel = lgb.train(params,\n                train_data,\n                20000,\n                valid_sets=[train_data, test_data],\n                callbacks=[lgb.early_stopping(stopping_rounds=500), lgb.log_evaluation(50)])","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:39:15.208754Z","iopub.execute_input":"2024-05-13T14:39:15.209170Z","iopub.status.idle":"2024-05-13T14:39:44.016802Z","shell.execute_reply.started":"2024-05-13T14:39:15.209139Z","shell.execute_reply":"2024-05-13T14:39:44.015411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = model.predict(test_set, num_iteration=model.best_iteration)\npreds","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:39:44.018166Z","iopub.execute_input":"2024-05-13T14:39:44.018484Z","iopub.status.idle":"2024-05-13T14:39:44.232773Z","shell.execute_reply.started":"2024-05-13T14:39:44.018457Z","shell.execute_reply":"2024-05-13T14:39:44.231681Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set[\"PredictedRevenue\"] = preds\ntest_set","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:39:44.234355Z","iopub.execute_input":"2024-05-13T14:39:44.234758Z","iopub.status.idle":"2024-05-13T14:39:44.718037Z","shell.execute_reply.started":"2024-05-13T14:39:44.234726Z","shell.execute_reply":"2024-05-13T14:39:44.716575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {'objective': 'regression',\n              'metric': 'rmse',\n              'num_leaves': 50,\n              'learning_rate': 0.01,\n              'bagging_fraction': 0.75,\n              'feature_fraction': 0.8,\n              'bagging_frequency': 5,\n              'force_row_wise': True,  \n              'verbose': -1,\n              'predict_disable_shape_check': True  \n}\n    \nlgb_train = lgb.Dataset(X_train, label=y_train)\nlgb_val = lgb.Dataset(X_test, label=y_test)\nmodel_1 = lgb.train(lgb_params, lgb_train, \n                  num_boost_round=700, \n                  valid_sets=[lgb_val], \n                  callbacks=[lgb.early_stopping(stopping_rounds=500), \n                             lgb.log_evaluation(50), ])","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:39:44.719356Z","iopub.execute_input":"2024-05-13T14:39:44.719705Z","iopub.status.idle":"2024-05-13T14:40:14.644474Z","shell.execute_reply.started":"2024-05-13T14:39:44.719676Z","shell.execute_reply":"2024-05-13T14:40:14.643542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_1 = model_1.predict(test_set.loc[:, test_set.columns != 'PredictedRevenue'], \n                          num_iteration=model.best_iteration, )\npreds_1","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:40:14.645777Z","iopub.execute_input":"2024-05-13T14:40:14.646667Z","iopub.status.idle":"2024-05-13T14:40:14.943715Z","shell.execute_reply.started":"2024-05-13T14:40:14.646635Z","shell.execute_reply":"2024-05-13T14:40:14.942577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set[\"PredictedRevenue\"] = preds_1\ntest_set","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:40:14.945359Z","iopub.execute_input":"2024-05-13T14:40:14.946097Z","iopub.status.idle":"2024-05-13T14:40:15.359280Z","shell.execute_reply.started":"2024-05-13T14:40:14.946055Z","shell.execute_reply":"2024-05-13T14:40:15.357942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from lightgbm import LGBMRegressor","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:40:15.360939Z","iopub.execute_input":"2024-05-13T14:40:15.361353Z","iopub.status.idle":"2024-05-13T14:40:15.366723Z","shell.execute_reply.started":"2024-05-13T14:40:15.361321Z","shell.execute_reply":"2024-05-13T14:40:15.365517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgb_params = {'num_leaves' : 50, 'learning_rate' : 0.01, \n              'bagging_fraction' : 0.75, 'feature_fraction' : 0.8, \n              'bagging_frequency' : 5, 'force_row_wise': [True], \n              'verbose': -1,}\n    \nmodel_2 = LGBMRegressor(**lgb_params, n_estimators=700)\nmodel_2.fit(X_train, y_train, \n          eval_set=[(X_test, y_test)])","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:40:15.368368Z","iopub.execute_input":"2024-05-13T14:40:15.368789Z","iopub.status.idle":"2024-05-13T14:41:45.254665Z","shell.execute_reply.started":"2024-05-13T14:40:15.368751Z","shell.execute_reply":"2024-05-13T14:41:45.253436Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds_2 = model_2.predict(test_set.loc[:, test_set.columns != 'PredictedRevenue'], \n                          num_iteration=model.best_iteration, )\npreds_2","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:41:45.258837Z","iopub.execute_input":"2024-05-13T14:41:45.259304Z","iopub.status.idle":"2024-05-13T14:41:45.663150Z","shell.execute_reply.started":"2024-05-13T14:41:45.259260Z","shell.execute_reply":"2024-05-13T14:41:45.661730Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_set[\"PredictedRevenue\"] = preds_2\ntest_set","metadata":{"execution":{"iopub.status.busy":"2024-05-13T14:41:45.665070Z","iopub.execute_input":"2024-05-13T14:41:45.665444Z","iopub.status.idle":"2024-05-13T14:41:46.080657Z","shell.execute_reply.started":"2024-05-13T14:41:45.665413Z","shell.execute_reply":"2024-05-13T14:41:46.078754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import joblib\n# Save the trained model to a file\njoblib.dump(model_2, 'model.pkl')\n","metadata":{"execution":{"iopub.status.busy":"2024-05-13T15:02:18.013076Z","iopub.execute_input":"2024-05-13T15:02:18.013529Z","iopub.status.idle":"2024-05-13T15:02:18.202530Z","shell.execute_reply.started":"2024-05-13T15:02:18.013496Z","shell.execute_reply":"2024-05-13T15:02:18.201351Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}