{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport json as js\nfrom pandas.io.json import json_normalize\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\nimport os\nprint(os.listdir(\"../input\"))\n\nimport sys\nimport warnings\n\nif not sys.warnoptions:\n    warnings.simplefilter(\"ignore\")\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Reading train and test data\ntrain_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e282b858a9ad803865907a5c2a47fabe02e2c07"},"cell_type":"code","source":"print(\"The shape of train data is :\",train_data.shape)\nprint(\"The shape of test data is:\",test_data.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f51bec8718de4f32825d16a80f081a4e70afb799"},"cell_type":"markdown","source":"Out of the 12 columns in the train dataset, 4 are json columns which we need to split in order to examine the complete dataset."},{"metadata":{"trusted":true,"_uuid":"c1a3e48051326040f90938d963ba6f2e2bf5f0a0"},"cell_type":"code","source":"# Retrieving the json data from the actual data\njson_cols = ['device','geoNetwork','totals','trafficSource']\njson_data = train_data.loc[:,json_cols]\njson_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3bfe936bf884e7d599d63dc74989e439f304e1f"},"cell_type":"code","source":"# A basic function to get all the features enclosed in the json columns\ndef jcol_feats(col_name):\n    j = js.loads(json_data[col_name][0])\n    feats = j.keys()\n    return feats","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"317fa8509bbb2d3736d5888d2991f733a159f185"},"cell_type":"code","source":"n_dev = len(jcol_feats('device'))\nprint(\"Number of features in device column:\",n_dev)\nn_geo= len(jcol_feats('geoNetwork'))\nprint(\"Number of features in geoNetwork column:\",n_geo)\nn_tot = len(jcol_feats('totals'))\nprint(\"Number of features in totals column:\",n_tot)\nn_traf = len(jcol_feats('trafficSource'))\nprint(\"Number of features in tafficSource column:\",n_traf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be615ed91d5d4fefb084b0d217a883bb31c5dc09"},"cell_type":"code","source":"# https://www.kaggle.com/shivamb/exploratory-data-analysis-ga-customer-revenue\n\njson_cols = ['device', 'geoNetwork', 'totals', 'trafficSource']\ndef load_df(filename):\n    path = \"../input/\" + filename\n    df = pd.read_csv(path, converters={column: js.loads for column in json_cols}, \n                     dtype={'fullVisitorId': 'str'})\n    \n    for column in json_cols:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}_{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91558ada362ada5d4e3a4cc4c0bb8865bd94d70b"},"cell_type":"code","source":"data = load_df(\"train.csv\")\ndata.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c718e9e18f46e5e57d008a3ea14c912c0601a1fb"},"cell_type":"code","source":"len(data['totals_transactionRevenue'].value_counts()) # Number of users from whom google earned revenue","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1e75b8d00e42911eaee8eead63dc435a4190c079"},"cell_type":"markdown","source":"**This shows that out of 0.9 million customers, just 5k customers are the part of revenue club for Google which is 0.59%, i.e. less than a percent of users**."},{"metadata":{"_uuid":"f7fdb3f6ee28d559580d0cd4856d5b33f070f999"},"cell_type":"markdown","source":"### Removing constant columns"},{"metadata":{"trusted":true,"_uuid":"10d5712b9589dbd0893e95739f5888218587394b"},"cell_type":"code","source":"# Simple function for removing constant columns\ndef const(d):\n    dt = d.loc[:,d.apply(pd.Series.nunique) != 1]\n    return dt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cf95d5401143f1b8c0c4f03e71125f3722c55d9"},"cell_type":"code","source":"act_data = const(data)\nact_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a4d53f82ec0a402f7cc6f3437ba3b97d4c30262b"},"cell_type":"markdown","source":"### Missing values"},{"metadata":{"trusted":true,"_uuid":"b19b79ecf37b8e46694852e67f480e0dcae7f365"},"cell_type":"code","source":"def missing_data(data):  #calculates missing values in each column\n    total = data.isnull().sum().reset_index()\n    total.columns  = ['Feature_Name','Missing_value']\n    total_val = total[total['Missing_value'] > 0]\n    total_val = total.sort_values(by ='Missing_value',ascending= False)\n    return total_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b188955edc5486c909057bc8cc83348b0ed33805"},"cell_type":"code","source":"missing_data(act_data).head(9) # 9 features in train data have missing value in them.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d36fbb627b1568ef7165d3df9abaef55e6db750a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}