{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"925ff14268b27b4098ea9351388453ef40471827"},"cell_type":"code","source":"#For Importing Libs\nimport json\nfrom pandas.io.json import json_normalize\nimport random\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.plotly as py\nimport plotly.graph_objs as go","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6626abbc58ebe822c0e08738432ca10e1cce965a"},"cell_type":"code","source":"# for each in list(train_data):\n#     print(\"Column: \",each,\"Dtype:\",train_data[each].dtype)\ntrain_data.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1c84104065db571baa6200a47c5665c7fe1c73a"},"cell_type":"code","source":"columns = ['device', 'geoNetwork', 'totals', 'trafficSource'] # Columns that have json format\n\ndir_path = \"../input/\" # you can change to your local \n\n# p is a fractional number to skiprows and read just a random sample of the our dataset. \np = 0.4 # *** In this case we will use 50% of data set *** #\n\n#Code to transform the json format columns in table\ndef json_read(df):\n    #joining the [ path + df received]\n    data_frame = dir_path + df\n    \n    #Importing the dataset\n    df = pd.read_csv(data_frame, \n                     converters={column: json.loads for column in columns}, # loading the json columns properly\n                     dtype={'fullVisitorId': 'str'}, # transforming this column to string\n                     skiprows=lambda i: i>0 and random.random() > p)# Number of rows that will be imported randomly\n    \n    for column in columns: #loop to finally transform the columns in data frame\n        #It will normalize and set the json to a table\n        column_as_df = json_normalize(df[column]) \n        # here will be set the name using the category and subcategory of json columns\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns] \n        # after extracting the values, let drop the original columns\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n        \n    # Printing the shape of dataframes that was imported     \n    print(f\"Loaded {os.path.basename(data_frame)}. Shape: {df.shape}\")\n    return df # returning the df after importing and transforming","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f84782f8f674290f13002b3244aadd76b824e623"},"cell_type":"code","source":"%%time \n# %%time is used to calculate the timing of code chunk execution #\ndf_train = json_read(\"train.csv\") \n# The same to test dataset\ndf_test = json_read(\"test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2941cee7e8a59a409c8fed9b2634ae151275ad5"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"31101b30b4671925e921394d1d6bdc63a8821aa6"},"cell_type":"code","source":"df_train.channelGrouping.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ecba579b263dda7f331659a47539e71f819c1b6"},"cell_type":"code","source":"df_train.channelGrouping.value_counts().plot(kind=\"bar\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63e6583d9c42aa15562d1bdbbc93db0e162a648f"},"cell_type":"code","source":"#dataset date range\nprint(min(df_train[\"date\"]),\"-\",max(df_train[\"date\"]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"38726f987d6d9acebf6c81c3f055bdb3575b7e8d"},"cell_type":"code","source":"#notice that\n# sessionId = fullVisitorId + \"_\" + visitId","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5090e07599af8fa6638490dae4ea21d600d59aca"},"cell_type":"code","source":"#vistors start time\ntimedf = pd.DataFrame()\ntimedf['visitStartTime'] = pd.to_datetime(df_test['visitStartTime'],unit='s')\ntimedf['timeHH'] = [str(x).split(' ')[1].split(':')[0] for x in timedf['visitStartTime']] # spliting HH from time stamp\nprint(timedf.head(2))\ntimedf['timeHH'].value_counts().sort_values(ascending=False).plot('bar',figsize=[12,8]) # maximum visit is during evening 17 - 18, count holds the value from time x:0:0 to x:59:59 ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2046e323a9ddcebb3c6296e64b8d69f122c144f8"},"cell_type":"code","source":"#top 10 device browsers\nprint(df_train['device.browser'].value_counts().sort_values(ascending=False)[0:10])\ndf_train['device.browser'].value_counts().sort_values(ascending=False)[0:10].plot('bar',figsize=[12,8])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eff9e0cde9a6ddf8350064cf77325917a0d47cb8"},"cell_type":"code","source":"df_train[\"device.deviceCategory\"].value_counts().plot(\"bar\",figsize=[10,6])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8abe9938abcea9eac7210faf9ac810ec51e589a"},"cell_type":"markdown","source":"### Top 20 DeviceCategory and Device Browser"},{"metadata":{"trusted":true,"_uuid":"8bfcc1a7de119226724d96b0dfcb5f0f1054d128"},"cell_type":"code","source":"DcDb = df_train[['visitId','device.deviceCategory','device.browser']].groupby(['device.browser','device.deviceCategory']).count().reset_index().sort_values(by = 'visitId',ascending=0)[0:20]\nDcDb.columns = ['device.browser','device.deviceCategory','BrowserUsageCount']\nDcDb.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"543bc8096b1cf561225d8be661a83920e8264549"},"cell_type":"code","source":"with sns.axes_style(style='ticks'):\n    g = sns.factorplot(\"device.deviceCategory\", \"BrowserUsageCount\", \"device.browser\", data=DcDb, kind=\"bar\",size=8,aspect=1.5)\n    g.set_axis_labels(\"device.deviceCategory\", \"BrowserUsageCount\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c3cdc40a71988e51a5021a664325c50a900456a6"},"cell_type":"code","source":"#Device category and device mobile type count\ndf_train[df_train['device.deviceCategory'] == 'desktop']['device.isMobile'].value_counts()\nDcDm = df_train[['fullVisitorId','device.deviceCategory','device.isMobile']].groupby(['device.deviceCategory','device.isMobile']).count().reset_index()\nDcDm.columns = ['device.deviceCategory','device.isMobile','VisitorCount']\nDcDm","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29377ad48c8ddeddf7fd6f5371edab34f3deccd0"},"cell_type":"markdown","source":"### TOp 20 Device browser and operating system used**"},{"metadata":{"trusted":true,"_uuid":"5c4fd25b58b8d629cb7c602e3c9d8314370a722a"},"cell_type":"code","source":"ndf = df_train[['visitId','device.operatingSystem','device.browser']].groupby(['device.browser','device.operatingSystem']).count().reset_index().sort_values(by = 'visitId',ascending=0)[0:20]\nndf.columns = ['device.browser','device.operatingSystem','OSUsageCount']\nndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0883b06b8482e7b55f0c4b4e0dbd7c86826a56f"},"cell_type":"code","source":"with sns.axes_style(style='ticks'):\n    g = sns.factorplot(\"device.operatingSystem\", \"OSUsageCount\", \"device.browser\", data=ndf, kind=\"bar\",size=6,aspect=2)\n    g.set_axis_labels(\"device.operatingSystem\", \"OSUsageCount\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0a0f74d2c484df92536fcc329f450e47cf302290"},"cell_type":"code","source":"#top 15 countries\ndf_train['geoNetwork.country'].value_counts()[0:15].plot('bar',figsize=[12,6])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9db605b19d87454d257c6140e83e8e0ea09cbd16"},"cell_type":"code","source":"#top 15 cities\ndf_train.loc[df_train['geoNetwork.city'] == 'not available in demo dataset']['geoNetwork.city'] = np.nan #replacing \"not available in demo dataset\" with nan\n#top 10 countries\ndf_train['geoNetwork.city'].value_counts()[0:15].plot('bar',figsize=[12,6])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d4e6190350aa4feca4682ef852e20ee1199fd47b"},"cell_type":"code","source":"df_train['geoNetwork.continent'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5bf9d37d0c8566263cdcf25bc7c1a50f83dd07bd"},"cell_type":"code","source":"#top conuntry in every continent\ntop_country_per_continent = df_train.groupby(['geoNetwork.continent','geoNetwork.country']).size().reset_index().sort_values(0,ascending = False).groupby(['geoNetwork.continent']).head(2)\ntop_country_per_continent.columns = ['geoNetwork.continent','geoNetwork.country','count']\n\ng = sns.factorplot(x=\"geoNetwork.country\", y ='count', hue='geoNetwork.continent',data=top_country_per_continent,kind=\"bar\", size=6, aspect=1.5)\ng.set_xticklabels(rotation=90, ha=\"right\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90b1b61686bc2d5276398d3b618f4d01a6d057f0"},"cell_type":"code","source":"#top subcontinent \ndfsubcont = df_train['geoNetwork.subContinent'].value_counts().reset_index()[0:15]\ndfsubcont.columns = ['geoNetwork.subContinent','count']\ng = sns.factorplot(x=\"geoNetwork.subContinent\", y ='count',data=dfsubcont,kind=\"bar\", size=6, aspect=1.5)\ng.set_xticklabels(rotation=90, ha=\"right\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05117dc524e2cefab651f46770ca5e1718a8f67c"},"cell_type":"code","source":"totaldf = df_train[['geoNetwork.country','totals.hits']].groupby('geoNetwork.country').count().reset_index().sort_values(by='totals.hits',ascending=False).reset_index().drop(['index'],axis = 1)\nplt.figure(figsize=(8,8))\np = plt.pie(totaldf['totals.hits'][0:10], labels=totaldf['geoNetwork.country'][0:10],autopct='%1.0f%%',pctdistance=0.6, labeldistance=1.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea910f922892c7dde12d6495fbb92a376fae8cdf"},"cell_type":"code","source":"#US vs rest of the world\nlabel = ['US','Others']\nvalue =[totaldf.loc[0]['totals.hits'],totaldf.loc[1:len(totaldf)][['totals.hits']].sum()]\nplt.figure(figsize=(8,8))\np1 = plt.pie(value, labels=label,autopct='%1.0f%%',pctdistance=0.6, labeldistance=1.1)\nprint(\"US alone has 40 percent of all hits\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a927bbe307077f4f6435e9e4a8307b69d256fc44"},"cell_type":"code","source":"#US vs rest of the world\nlabel = ['US + India','Others']\nvalue =[totaldf.loc[0:2][['totals.hits']].sum(),totaldf.loc[2:len(totaldf)][['totals.hits']].sum()]\nplt.figure(figsize=(8,8))\np1 = plt.pie(value, labels=label,autopct='%1.1f%%',pctdistance=0.6, labeldistance=1.1)\nprint(\"US + India alone has around 50 percent of all hits\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73e21ce51445f55d5f496bcc6230687e42e20d6e"},"cell_type":"code","source":"dftss = df_train['trafficSource.source'].value_counts().reset_index()[0:9]\ndftss.columns = ['trafficSource.source','count']\nplt.figure(figsize=(9,9))\np1 = plt.pie(dftss['count'], labels=dftss['trafficSource.source'],autopct='%1.1f%%',pctdistance=0.6, labeldistance=1.1)\nprint(\"Top 6 traffic Source\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63893a7d226214d116a4407daac9400de4c92e45"},"cell_type":"code","source":"df_train[['trafficSource.source','geoNetwork.country']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"280234091b7bb840de0276874653052cf7da325d"},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"754f63bd77502b678bc2caf532aea117cb7d9063"},"cell_type":"code","source":"#printing nan counts of the columns\nfor column in df_train.columns:\n    if(df_train[column].isnull().values.any()):\n        print(column,\"has Nan Count: \",df_train[column].isna().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34e8a19f31504fa7da6dc55a1439e3947a845fd4"},"cell_type":"code","source":"df_train['totals.bounces'].isna().sum()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}