{"cells":[{"metadata":{"_uuid":"bb32cc5d2722e96bb02725aaece54c5509bdacb9"},"cell_type":"markdown","source":"**Objective of the notebook:**\n\nIn this notebook, let us explore the given dataset and make some inferences on the way.\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np\nimport pandas as pd\nfrom pandas.io.json import json_normalize\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n\n\nimport matplotlib.pylab as plt\nimport seaborn as sns\ncolor = sns.color_palette()\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.offline as offline\noffline.init_notebook_mode()\nimport plotly.tools as tls\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e08fe7f8ec7419ba476cd52664140ed6ae736a8c"},"cell_type":"code","source":"train_df = load_df()\ntest_df = load_df(\"../input/test.csv\")\ntrain_df.shape,test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"302c2b9449b6173ed44ad1d27d3547573368afd4"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dcaa5e0839118e5e2d9192f450569309e8c3798d"},"cell_type":"markdown","source":"There are two columns  exist in train dataset  but not in test dataset. Let's find these two features."},{"metadata":{"trusted":true,"_uuid":"103606eecab848cec2f409866fb1bf8bc05b733b"},"cell_type":"code","source":"intersection = list(set(train_df.columns) & set(test_df.columns))\nli = list(set(train_df.columns) -set(intersection))\n\nprint ('feature not in test but in train:' ,li)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"27ed7b4f8d6542413a9e549b68dfbc85687111c9"},"cell_type":"markdown","source":"Here, **totals.transactionRevenue** is a target feature.\nAnd, **trafficSource.campaignCode** is a redundant feature and not useful for prediction hence, remove this feature from train dataset."},{"metadata":{"trusted":true,"_uuid":"5a1da83e94020c61b718428d99a925daa7315733"},"cell_type":"code","source":"train_df.drop('trafficSource.campaignCode' , axis =1,inplace = True)\ntrain_df.shape,test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4f0473a7ce96b1d3d79230b793208211823eed4"},"cell_type":"markdown","source":"** Missing data : **"},{"metadata":{"trusted":true,"_uuid":"0c153951596f02945f8af0201193cc75955bfb1e"},"cell_type":"code","source":"def check_missing(df):\n    \n    total = df.isnull().sum().sort_values(ascending = False)\n    percent = (df.isnull().sum()/df.isnull().count()*100).sort_values(ascending = False)\n    missing_data  = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n    return missing_data\n    \n\nmissing_data_df = check_missing(train_df)\nmissing_data_test = check_missing(test_df)\n\nprint('Missing data in train set: \\n' , missing_data_df.head(10))\nprint('\\nMissing data in test set: \\n'  ,missing_data_test.head(10))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5c462ae5d9f2a9514078b167c89bc7de2042528"},"cell_type":"markdown","source":"**Redundant features :** Let's consider the feature which has a constant value as a redundant feature. And remove these redundant features"},{"metadata":{"trusted":true,"_uuid":"019b1657e781a8605a018c01cc09ac6a0af2d2bf"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"343c68d801a55d17dc08bbaa35192a588cd94a7b"},"cell_type":"code","source":"def find_uni(df):\n    col_list = df.columns\n    redundant_col =[]\n    for col in col_list:\n        if df[col].nunique() == 1:\n            redundant_col.append(col)\n    return redundant_col\n\n\nredundant_col_train  = find_uni(train_df)\nredundant_col_test = find_uni(test_df)\n\nprint ('Number of redundant features in train data :',len(redundant_col_train))\nprint ('Redundant Feature :', redundant_col_train)\n\nprint ('\\n Number of redundant features in test data :',len(redundant_col_test))\nprint ('Redundant Feature :', redundant_col_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72eb9bbebc8c398dda3735d0199c28b61d1a40ca"},"cell_type":"code","source":"intersection = list(set(redundant_col_train) & set(redundant_col_test))\n\ntrain_df.drop(intersection, axis =1, inplace = True)\ntest_df.drop(intersection, axis =1, inplace = True)\ntrain_df.shape,test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5d3d79c5169bd32c43ccbfe2287fcba0a101c398"},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70252dc3d807e8a9d9e0dccc63d2a1379836357d"},"cell_type":"markdown","source":"Let's explore date feature:"},{"metadata":{"trusted":true,"_uuid":"81f443b17d20e01bfb6c8ea029624f01cc9f163a"},"cell_type":"code","source":"import datetime\n\ntrain_df['date'] = train_df['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\ntrain_df['date'] = pd.to_datetime(train_df['date'])\nprint ('train_data:', train_df['date'].describe())\n\ntest_df['date'] = test_df['date'].apply(lambda x: datetime.date(int(str(x)[:4]), int(str(x)[4:6]), int(str(x)[6:])))\ntest_df['date'] = pd.to_datetime(test_df['date'])\nprint ('\\n test data:', test_df['date'].describe())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f2a774287e51b2f98adec6eaac492b861234825"},"cell_type":"markdown","source":"**Inferences:**\n\n* train dataset have 1 year data from 1st Aug, 2016 to 1st Aug,  2017.\n* Test data have 9 month data from 2nd Aug, 2017 to 30th April, 2018."},{"metadata":{"_uuid":"dd071e56c54fe9d5d3e556ea98c317f03e813c7c"},"cell_type":"markdown","source":"Let's plot **channelGrouping** feature"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"c7e14f88a6716ed0dde128b169500f96508a3542"},"cell_type":"code","source":"temp = train_df['channelGrouping'].value_counts()\nlabels = temp.index\nsizes = (temp / temp.sum())*100\ntrace = go.Pie(labels=labels, values=sizes, hoverinfo='label+percent')\nlayout = go.Layout(title='channelGrouping')\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"31168f94e6a782e14a1f5972a6de5b8f88a109fd"},"cell_type":"code","source":"temp = train_df['device.browser'].value_counts()\nlabels = temp.index\nsizes = (temp / temp.sum())*100\ntrace = go.Pie(labels=labels, values=sizes, hoverinfo='label+percent')\nlayout = go.Layout(title='device.browser')\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"71f3e5b15462649e0e0d77c11fa9e9640021ba80"},"cell_type":"code","source":"temp = train_df['device.deviceCategory'].value_counts()\nlabels = temp.index\nsizes = (temp / temp.sum())*100\ntrace = go.Pie(labels=labels, values=sizes, hoverinfo='label+percent')\nlayout = go.Layout(title='device.deviceCategory')\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5df38cb2211010ab5519bd7c7e0d3c53abc7444"},"cell_type":"code","source":"for col in ['visitNumber', 'totals.hits', 'totals.pageviews', 'totals.transactionRevenue']:\n    train_df[col] = train_df[col].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b17878a2bce0f0653e2f07ecb437d4718257527f"},"cell_type":"code","source":"plt.hist(np.log(train_df.loc[train_df['totals.transactionRevenue'].isna() == False, 'totals.transactionRevenue']));\nplt.title('Distribution of revenue');","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e2ed97686a372b89e874864f271765cdcbe9129b"},"cell_type":"markdown","source":"**More to come. Stay tuned.!**"},{"metadata":{"_uuid":"24f8284cc41e9cd95dff59e5f38e3784b822332f"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"1c4e6ee7d07b9bc675ab31d7bb05496bd94a951b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}