{"cells":[{"metadata":{"_uuid":"43fc51171c646421e40517e865ef35d2ab18f749"},"cell_type":"markdown","source":"**<font size=5>Objectives:<font>**\n<font size=3>\n* Explore importance of leak data in the whole data set.  \n*     What is the % missing 'totals.transactionRevenue' in leak data row.  \n*     What is the % of frequence in leak data row.  \n*     What is the correlation of importance feature between 'totals.transactionRevenue' <font>\n    \n**<font size=3>I would try my best hope you like it<font>**\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true,"_kg_hide-input":false},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\n\n# DRAGONS\nimport xgboost as xgb\nimport lightgbm as lgb\nimport catboost as cat\n\n# plots\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\n# pandas / plt options\npd.options.display.max_columns = 999\nplt.rcParams['figure.figsize'] = (14, 7)\nfont = {'family' : 'verdana',\n        'weight' : 'bold',\n        'size'   : 14}\nplt.rc('font', **font)\n\n# remove warnings\nimport warnings\nwarnings.simplefilter(\"ignore\")\n\n# garbage collector\nimport gc\ngc.enable()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"96246a26954e661fb855357e75653530764a5e84"},"cell_type":"markdown","source":"**<font size=5>load data</font>**"},{"metadata":{"trusted":true,"_uuid":"5f6cf8b58c23fa10a8b0eaeed67dd65f63c101d2"},"cell_type":"code","source":"import os\nprint(os.listdir(\"../input\"))\n\ntrain = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_train.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})\ntest = pd.read_csv('../input/create-extracted-json-fields-dataset/extracted_fields_test.gz', dtype={'date': str, 'fullVisitorId': str, 'sessionId':str, 'visitId': np.int64})\ntrain.shape, test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22b94e464bb72752eca93c2c5fc8cafd8fcd5092"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"595bcbfd11b1eb8f43d4c89658d7a86c3bcfe365"},"cell_type":"code","source":"traincolumns_1 = train.columns\ntraincolumns_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ba797416f93b4537d92fa5417e379d8f4853daf"},"cell_type":"code","source":"train_store_1 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntrain_store_2 = pd.read_csv('../input/exported-google-analytics-data/Train_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_1 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ntest_store_2 = pd.read_csv('../input/exported-google-analytics-data/Test_external_data_2.csv', low_memory=False, skiprows=6, dtype={\"Client Id\":'str'})\ndataset = pd.concat(objs=[train_store_1, train_store_2], axis=0)\ndataset.info()\ndel dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffc9dc71a7b5df54a7fd80be5b09f0a48de2a8b2"},"cell_type":"code","source":"for df in [train_store_1, train_store_2, test_store_1, test_store_2]:\n    df[\"visitId\"] = df[\"Client Id\"].apply(lambda x: x.split('.', 1)[1]).astype(np.int64)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3325857e3165ecccdb16f37608d830ad1f9dd614"},"cell_type":"markdown","source":"**<font size=5>Check extend feature</font>**"},{"metadata":{"trusted":true,"_uuid":"45d891d983cf5f71d4404ffecb16bee78dd4ad8c"},"cell_type":"code","source":"train = train.merge(pd.concat([train_store_1, train_store_2], sort=False), how=\"left\", on=\"visitId\")\ntest = test.merge(pd.concat([test_store_1, test_store_2], sort=False), how=\"left\", on=\"visitId\")\n\n# Drop Client Id\nfor df in [train, test]:\n    df.drop(\"Client Id\", 1, inplace=True)\nleakcolumns = [x for x in train.columns if x not in traincolumns_1]\nleakcolumns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1bf892c6a1ef5fc6db9d3ea0ea85997dec45fe50"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c32abdb18e8df1067090fb14d89dec3640a4e49e"},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3c4dca48b1664ff0dc16efc50b2994ec84aef29f"},"cell_type":"markdown","source":"\n**<font size=5>Processed feature for further use</font>**"},{"metadata":{"trusted":true,"_uuid":"d0865a41e55dcc74e137d2675edb1762a64e870f"},"cell_type":"code","source":"train['has_revenue'] = train['totals.transactionRevenue'].apply(lambda x: 1 if x > 0 else 0)\n\nfor df in [train, test]:\n    df['browser.os'] = df['device.browser'] + '_' + df['device.operatingSystem']\n\nfor df in [train, test]:\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['sess_date_dow'] = df['date'].dt.dayofweek\n    df['sess_date_hours'] = df['date'].dt.hour\n    df['sess_date_dom'] = df['date'].dt.day\n    df.sort_values(['fullVisitorId', 'date'], ascending=True, inplace=True)\n    df['next_session_1'] = (\n        df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(1)\n    ).astype(np.int64) // 1e9 // 60 // 60\n    df['next_session_2'] = (\n        df['date'] - df[['fullVisitorId', 'date']].groupby('fullVisitorId')['date'].shift(-1)\n    ).astype(np.int64) // 1e9 // 60 // 60","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8a07b5bcf7ff08dbe6244135bab32096f3b5c257"},"cell_type":"markdown","source":"**<font size=5>Knowing the missing values</font>**"},{"metadata":{"trusted":true,"_uuid":"36e81a33a979f3c4cea36e0e9541012c79747ab9","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def missing_values(data):\n    print(data.shape)\n    total = data.isnull().sum().sort_values(ascending = False) # getting the sum of null values and ordering\n    percent = (data.isnull().sum() / data.isnull().count() * 100 ).sort_values(ascending = False) #getting the percent and order of null\n    df = pd.concat([total, percent], axis=1, keys=['Total', 'Percent']) # Concatenating the total and percent\n    print(\"Total columns at least one Values: \")\n    print (df[~(df['Total'] == 0)]) # Returning values of nulls different of 0\n    \nmissing_values(train) \n\nprint(\"\\n Total of Sales % of Total: \", round((train[train['totals.transactionRevenue'] != \\\n        np.nan]['totals.transactionRevenue'].count() / len(train['totals.transactionRevenue']) * 100),4))\n\n\ntrain_leak = train[train['Sessions'].isnull().values==False]\n\nmissing_values(train_leak) \n\nprint(\"\\n Total of Sales % of train_leak: \", round((train_leak[train_leak['totals.transactionRevenue'] != \\\n        np.nan]['totals.transactionRevenue'].count() / len(train_leak['totals.transactionRevenue']) * 100),4))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e7bba7697c98167bf76aa3991dcd1ac9630ff894"},"cell_type":"markdown","source":"<font size=3>\nWhole data row 1.2744% have 'totals.transactionRevenue' value  \n    \n Leak data row 42.4326% have 'totals.transactionRevenue' value\n</font>\n"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0a9875310b69e433177962edd5b4c7d8fcbecc91"},"cell_type":"code","source":"test.info()\nmissing_values(test) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5704c328a888b926a18abe90b5f8bb34fce2abf"},"cell_type":"markdown","source":"**<font size=5>Distribuition of transactions Revenues</font>**"},{"metadata":{"trusted":true,"_uuid":"e3af5f348695eb7e1d4cae9d9cb37ae9fa4dfa5d","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"for df in [train_store_1, train_store_2, test_store_1, test_store_2]:\n    del df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5dee3142107c89e4fef5b8ff01fdec0b69f133da","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def plotdisturbtion(df_train):\n    # Printing some statistics of our data\n    print(\"Transaction Revenue Min Value: \", \n          df_train[df_train['totals.transactionRevenue'] > 0][\"totals.transactionRevenue\"].min()) # printing the min value\n    print(\"Transaction Revenue Mean Value: \", \n          df_train[df_train['totals.transactionRevenue'] > 0][\"totals.transactionRevenue\"].mean()) # mean value\n    print(\"Transaction Revenue Median Value: \", \n          df_train[df_train['totals.transactionRevenue'] > 0][\"totals.transactionRevenue\"].median()) # median value\n    print(\"Transaction Revenue Max Value: \", \n          df_train[df_train['totals.transactionRevenue'] > 0][\"totals.transactionRevenue\"].max()) # the max value\n\n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    ax = sns.distplot(np.log(df_train[df_train['totals.transactionRevenue'] > 0][\"totals.transactionRevenue\"] + 0.01), bins=40, kde=True)\n    ax.set_xlabel('Transaction RevenueLog', fontsize=15) #seting the xlabel and size of font\n    ax.set_ylabel('Distribuition', fontsize=15) #seting the ylabel and size of font\n    ax.set_title(\"Distribuition of Revenue Log\", fontsize=20) #seting the title and size of font\n\n    plt.subplot(1,2,2)\n    plt.scatter(range(df_train.shape[0]), np.sort(df_train['totals.transactionRevenue'].values))\n    plt.xlabel('Index', fontsize=15) # xlabel and size of words\n    plt.ylabel('Revenue value', fontsize=15) # ylabel and size of words\n    plt.title(\"Revenue Value Distribution\", fontsize=20) # Setting Title and fontsize\n    plt\n    \nprint('whole data')\nplotdisturbtion(train)\nprint('leak data row')\nplotdisturbtion(train_leak)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3a3513a4a3ef3beb8e1721f5f7a0a421828c4f8"},"cell_type":"markdown","source":"**<font size=5>Distribuition of Category columns</font>**"},{"metadata":{"trusted":true,"_uuid":"bcd76abddad74e3408c3325b6275818aba9dffce","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"from plotly.offline import init_notebook_mode, iplot\nimport plotly.graph_objs as go\nfrom plotly import tools\nimport warnings\nwarnings.simplefilter(action='ignore', category=FutureWarning)\ninit_notebook_mode(connected=True)\n\ndef barplot_percentage(count_feat, color1= 'green', \n                       color2= 'rgb(26, 118, 255)',color3= 'red',num_bars= None):\n\n    train_channel = 100*train[train[count_feat].isin(train[count_feat]\\\n            .value_counts()[:7].index.values)][count_feat].value_counts()/len(train)\n    train_channel = train_channel.to_frame().reset_index()\n\n    test_channel = 100*test[test[count_feat].isin(train[count_feat]\\\n            .value_counts()[:7].index.values)][count_feat].value_counts()/len(test)\n    test_channel = test_channel.to_frame().reset_index()\n    \n    leak_channel = 100*train_leak[train_leak[count_feat].isin(train[count_feat]\\\n            .value_counts()[:7].index.values)][count_feat].value_counts()/len(train_leak)\n    leak_channel = leak_channel.to_frame().reset_index()\n    \n    if num_bars:\n        train_channel = train_channel.head(num_bars)\n        test_channel = test_channel.head(num_bars)\n        leak_channel = leak_channel.head(num_bars)\n\n    trace0 = go.Bar(\n        x=train_channel['index'],\n        y=train_channel[count_feat],\n        name='Train set',\n        marker=dict(color=color1)\n    )\n    trace1 = go.Bar(\n        x=test_channel['index'],\n        y=test_channel[count_feat],\n        name='Test set',\n        marker=dict(color=color2,)\n    )\n    trace2 = go.Bar(\n        x=leak_channel['index'],\n        y=leak_channel[count_feat],\n        name='leak data set',\n        marker=dict(color=color3,)\n    )\n\n    layout = go.Layout(\n        height=400,\n        title='{} grouping'.format(count_feat),\n        xaxis=dict(\n            tickfont=dict(size=14, color='rgb(107, 107, 107)')\n        ),\n        yaxis=dict(\n            title='Percentage of visits',\n            titlefont=dict(size=16, color='rgb(107, 107, 107)'),\n            tickfont=dict(size=14, color='rgb(107, 107, 107)')\n        ),\n        legend=dict(\n            x=1.0,\n            y=1.0,\n            bgcolor='rgba(255, 255, 255, 0)',\n            bordercolor='rgba(255, 255, 255, 0)'\n        ),\n        barmode='group',\n        bargap=0.15,\n        bargroupgap=0.1\n    )\n\n    fig = go.Figure(data=[trace0, trace1,trace2], layout=layout)\n    iplot(fig)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91902c662f83da0e13643b014e7ba5388bbdd873","_kg_hide-output":false},"cell_type":"code","source":"for x in ['geoNetwork.country','geoNetwork.region','geoNetwork.metro','channelGrouping','browser.os']:\n    barplot_percentage(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7366a30025fcbb890c3b954ef0e5c130598eedec"},"cell_type":"code","source":"for x in ['geoNetwork.networkDomain','trafficSource.medium','device.browser']:\n    barplot_percentage(x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"357f5618de27ecbd3bc2a4f1ea26a47445ca7407"},"cell_type":"markdown","source":"**<font size=5>Distribuition and Trend of Number columns</font>**"},{"metadata":{"trusted":true,"_uuid":"696a484ca8a9d1b9affdb6bfae39819c10fd6e33"},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ac881184f4772afec0e8b655015f5ea1f975205"},"cell_type":"code","source":"print(train['has_revenue'].unique())\n# print(train['has_revenue'])\ntrain['totals.transactionRevenue'] = train['totals.transactionRevenue'].fillna(0)\ntrain_leak['totals.transactionRevenue'] = train_leak['totals.transactionRevenue'].fillna(0)\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e651630671f651c1b51798343a6f1c77c2abca3"},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10644636a795997fce5bf0d3dbd2e41215ede6c3","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def plotrevenues(col_1,col_2):\n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    plt.title(\"Number of \" + col_1 + \" and revenue\")\n    ax = sns.scatterplot(x=col_1 , y='totals.transactionRevenue',\n                     data=train,color='orange', hue='has_revenue')\n    plt.subplot(1,2,2)\n    plt.title(\"Number of \" + col_2 + \"and revenue\")\n    ax = sns.scatterplot(x=col_2, y='totals.transactionRevenue',\n                     data=train,color='orange', hue='has_revenue')\n    plt\n    \n    cnt_col1 = train.groupby(col_1)['totals.transactionRevenue'].agg(['mean','count'])\n    cnt_col2= train.groupby(col_2)['totals.transactionRevenue'].agg(['mean','count'])\n    \n    cnt_col1 = cnt_col1.reset_index()\n    cnt_col2 = cnt_col2.reset_index()\n    \n    cnt_col1 = cnt_col1[cnt_col1['count']>10]\n    cnt_col2 = cnt_col2[cnt_col2['count']>10]\n    \n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    plt.title(\"Number of \" + col_1 + \" and mean revenue\")\n    ax = sns.scatterplot(x=col_1, y='mean',\n                     data=cnt_col1,color='blue')\n    \n    plt.subplot(1,2,2)\n    plt.title(\"Number of \" + col_2 + \"and mean revenue\")\n    ax = sns.scatterplot(x=col_2, y='mean',\n                     data=cnt_col2,color='blue')\n    plt\n\nplotrevenues('totals.pageviews','totals.hits')\n# plotrevenues('next_session_1','next_session_2')\nplotrevenues('visitNumber','sess_date_dow')\n# plotrevenues('sess_date_hours','sess_date_dom')\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d997c6c45c2eeaf55d194fe0aa55ba055dea042"},"cell_type":"markdown","source":"**<font size=3>Wow totals.pageviews totals.hits highly correlated with revenue</font>**"},{"metadata":{"_uuid":"214145c9ce43d4c36a732d7c5bba245a00318d0e"},"cell_type":"markdown","source":"**<font size=3>Filter feature session1 and session2</font>**"},{"metadata":{"trusted":true,"_uuid":"e7dc1f1bbe4cd010a747e90fdec881ac273c23d8","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"train_session_filter = train[train['next_session_1']>-500000]\ntrain_session_filter = train_session_filter[train_session_filter['next_session_2']>-500000]\ntrain_session_filter = train_session_filter[train_session_filter['totals.transactionRevenue'] < 100000000]\ndef plotrevenues_filter(col_1,col_2):\n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    plt.title(\"Number of \" + col_1 + \" and revenue\")\n    ax = sns.scatterplot(x=col_1 , y='totals.transactionRevenue',\n                     data=train_session_filter,color='orange', hue='has_revenue')\n    \n#     plt.figure(figsize=(14,5))\n    plt.subplot(1,2,2)\n    plt.title(\"Number of \" + col_2 + \"and revenue\")\n    ax = sns.scatterplot(x=col_2, y='totals.transactionRevenue',\n                     data=train_session_filter,color='orange', hue='has_revenue')\n    plt\n    \n    \n    t_s_f = train_session_filter[[col_1,col_2,'totals.transactionRevenue']]\n    \n    t_s_f[col_1] = t_s_f[col_1].apply(lambda x: int(x/10))\n    t_s_f[col_2] = t_s_f[col_2].apply(lambda x: int(x/10))\n    \n    \n    cnt_col1 = t_s_f.groupby(col_1)['totals.transactionRevenue'].agg(['mean','count'])\n    cnt_col2= t_s_f.groupby(col_2)['totals.transactionRevenue'].agg(['mean','count'])\n    \n    cnt_col1 = cnt_col1.reset_index()\n    cnt_col2 = cnt_col2.reset_index()\n    \n    cnt_col1 = cnt_col1[cnt_col1['count']>5]\n    cnt_col2 = cnt_col2[cnt_col2['count']>5]\n    \n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    plt.title(\"Number of \" + col_1 + \" and mean revenue\")\n    ax = sns.scatterplot(x=col_1, y='mean',\n                     data=cnt_col1,color='blue')\n    \n    plt.subplot(1,2,2)\n    plt.title(\"Number of \" + col_2 + \"and mean revenue\")\n    ax = sns.scatterplot(x=col_2, y='mean',\n                     data=cnt_col2,color='blue')\n    plt\n    \nplotrevenues_filter('next_session_1','next_session_2')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9824b889c218ba4271f02b51f414cba94cb2cc02"},"cell_type":"markdown","source":"**<font size=5>Distribuition and Trend of leak Number columns</font>**"},{"metadata":{"trusted":true,"_uuid":"092762b523b63b4f5062f3977c64e51cebc0a8b1"},"cell_type":"code","source":"for df in [train_leak]:\n    df[\"Revenue\"].fillna('$', inplace=True)\n    df[\"Revenue\"] = df[\"Revenue\"].apply(lambda x: x.replace('$', '').replace(',', ''))\n    df[\"Revenue\"] = pd.to_numeric(df[\"Revenue\"], errors=\"coerce\")\n    df[\"Revenue\"].fillna(0.0, inplace=True)\n    \nfor df in [train_leak]:\n    df[\"Avg. Session Duration\"][df[\"Avg. Session Duration\"] == 0] = \"00:00:00\"\n    df[\"Avg. Session Duration\"] = df[\"Avg. Session Duration\"].str.split(':').apply(lambda x: int(x[0]) * 60 + int(x[1]))\n    df[\"Bounce Rate\"] = df[\"Bounce Rate\"].astype(str).apply(lambda x: x.replace('%', '')).astype(float)\n    df[\"Goal Conversion Rate\"] = df[\"Goal Conversion Rate\"].astype(str).apply(lambda x: x.replace('%', '')).astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79e813f68a3d342661e25992f8b6ac46c0c34998"},"cell_type":"code","source":"leakcolumns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75450670dbbd71000bc0413cd692fa370bddec2f","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def plotrevenues_leak(col_1,col_2):\n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    plt.title(\"Number of \" + col_1 + \" and revenue\")\n    ax = sns.scatterplot(x=col_1 , y='totals.transactionRevenue',\n                     data=train_leak_filter,color='orange', hue='has_revenue')\n    \n#     plt.figure(figsize=(14,5))\n    plt.subplot(1,2,2)\n    plt.title(\"Number of \" + col_2 + \"and revenue\")\n    ax = sns.scatterplot(x=col_2, y='totals.transactionRevenue',\n                     data=train_leak_filter,color='orange', hue='has_revenue')\n    plt\n        \n    cnt_col1 = train_leak_filter.groupby(col_1)['totals.transactionRevenue'].agg(['mean','count'])\n    cnt_col2= train_leak_filter.groupby(col_2)['totals.transactionRevenue'].agg(['mean','count'])\n    \n    cnt_col1 = cnt_col1.reset_index()\n    cnt_col2 = cnt_col2.reset_index()\n    \n    cnt_col1 = cnt_col1[cnt_col1['count']>5]\n    cnt_col2 = cnt_col2[cnt_col2['count']>5]\n    \n    plt.figure(figsize=(14,5))\n    plt.subplot(1,2,1)\n    plt.title(\"Number of \" + col_1 + \" and mean revenue\")\n    ax = sns.scatterplot(x=col_1, y='mean',\n                     data=cnt_col1,color='blue')\n    \n    plt.subplot(1,2,2)\n    plt.title(\"Number of \" + col_2 + \"and mean revenue\")\n    ax = sns.scatterplot(x=col_2, y='mean',\n                     data=cnt_col2,color='blue')\n    plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"425707ebca18b09ce125a9b9d641751ca57fd7a5"},"cell_type":"markdown","source":"**<font size=3>Remove session > 200</font>**"},{"metadata":{"trusted":true,"_uuid":"8874a4983c0be96d8352a0a21fe340b319bf83aa"},"cell_type":"code","source":"train_leak_filter = train_leak[train_leak['Sessions']<200] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2be6b3b8fb69952d26777f5b6a3466f5c209dedd"},"cell_type":"code","source":"plotrevenues_leak('Sessions','Avg. Session Duration')\nplotrevenues_leak('Bounce Rate','Revenue')\nplotrevenues_leak('Transactions','Goal Conversion Rate')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3215bf2f7dc1efa8aee71cf4bc92f0c572e12d1"},"cell_type":"markdown","source":"**<font size=3>leak data all highly correlated to revenue</font>**"},{"metadata":{"_uuid":"4ee0c8f2c7de527411bd2b7b1f37102a8125e91a"},"cell_type":"markdown","source":"**<font size=5>Process some category columns</font>**"},{"metadata":{"trusted":true,"_uuid":"710b858bfacdf3ec5dbce8c3fbff3b35579ab703","_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"def topcolumn_proces(colname):\n    browsers_top= train[train[colname].isin(train[colname].value_counts()[:6].index.values)][colname]\n    browsercolumns = browsers_top.unique()\n    def source_mapping(x):\n        if x in browsercolumns:\n            return x\n        else:\n            return 'others'\n    train[colname+'_new'] = train[colname].map(lambda x:source_mapping(str(x))).astype('str')\n    train_leak[colname+'_new'] = train_leak[colname].map(lambda x:source_mapping(str(x))).astype('str')\nfor x in ['geoNetwork.country','channelGrouping','browser.os','trafficSource.medium']:\n    topcolumn_proces(x)    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2180db6e49bb7f0602e0c3033c5c763afb0b382"},"cell_type":"markdown","source":"**<font size=3>Filter revenue > 5000000000</font>**"},{"metadata":{"trusted":true,"_uuid":"d9a556e39e1452bc904805bd8cfa76c1c7c8457f"},"cell_type":"code","source":"train_Revenue_filter = train[train['totals.transactionRevenue']<5000000000]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63d4d3693c56876745d50f75ee82ed0d427a8e84"},"cell_type":"markdown","source":"**<font size=5>Plot jointplot plot</font>**"},{"metadata":{"trusted":true,"_uuid":"7c9e65b2e8bf28ada71880e9d2ad05413b80057f","_kg_hide-input":true},"cell_type":"code","source":"# train_n = train[train[\"Bounce Rate\"] > 0]\ndef plotjointplot(df,colname,cate_name):\n    g = sns.jointplot(df[colname], df['totals.transactionRevenue'],  s=1, size=12)\n    g.ax_joint.cla()\n    plt.sca(g.ax_joint)\n    categorycolumns = train[cate_name+'_new'].unique()\n    for cate_col in categorycolumns:\n        v = df[df[cate_name+'_new'] == cate_col]\n        plt.scatter(v[colname], v['totals.transactionRevenue'], s=4, label='{}'.format(cate_col))\n    plt.xlabel(colname,fontsize=15)\n    plt.ylabel('totals.transactionRevenue',fontsize=15)\n    plt.legend()\n    plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87c0d8aaea7cde65d71d5711f3e87e6a1f271621"},"cell_type":"markdown","source":"**<font size=3>Filter pageviews hits visitnumber</font>**"},{"metadata":{"trusted":true,"_uuid":"3daaca28c538482cd0a2b83e80df3b003623f963"},"cell_type":"code","source":"train_Ru_Pg_filter = train_Revenue_filter[train_Revenue_filter['totals.pageviews']<200]\ntrain_Ru_hits_filter = train_Revenue_filter[train_Revenue_filter['totals.hits']<250]\ntrain_Ru_vis_filter = train_Revenue_filter[train_Revenue_filter['visitNumber']<100]\n\n# for x in ['geoNetwork.country','channelGrouping','browser.os','trafficSource.medium']:\n#     plotjointplot(train_Ru_Pg_filter,'totals.pageviews',x)\nplotjointplot(train_Ru_Pg_filter,'totals.pageviews','geoNetwork.country')\nplotjointplot(train_Ru_hits_filter,'totals.hits','browser.os')\nplotjointplot(train_Ru_vis_filter,'visitNumber','geoNetwork.country')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"78cb468eedf29b11dcd0a056ab3dff91b9e650e5"},"cell_type":"markdown","source":"**<font size=3>Filter  leak data revenue and sessions</font>**"},{"metadata":{"trusted":true,"_uuid":"61111ef3bb40e3c77bae9ff7e44832c1e94fde0d"},"cell_type":"code","source":"train_leak_filter = train_leak[train_leak['totals.transactionRevenue']<200000000]\ntrain_leak_filter = train_leak_filter[train_leak_filter['Sessions']<200] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58cd1a7298e60d543f4e987e404478b93c509a62"},"cell_type":"code","source":"for x in ['browser.os']:\n    for y in leakcolumns:\n        plotjointplot(train_leak_filter,y,x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f315dc71173f8919378dc22efcc6257bfa4dcf65"},"cell_type":"markdown","source":"**<font size=5>Geolocation plot to visually understand the data</font>**  \n\n\n**<font size=3>whole data set :  vistis</font>**"},{"metadata":{"trusted":true,"_uuid":"6377dd5386db4a5d358bfe59f7f060d2b0bd38ee","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"def plotmapvisit(df_train):\n    # Counting total visits by countrys\n    countMaps = pd.DataFrame(df_train['geoNetwork.country'].value_counts()).reset_index()\n    countMaps.columns=['country', 'counts'] #renaming columns\n    countMaps = countMaps.reset_index().drop('index', axis=1) #reseting index and droping the column\n\n    data = [ dict(\n            type = 'choropleth',\n            locations = countMaps['country'],\n            locationmode = 'country names',\n            z = countMaps['counts'],\n            text = countMaps['country'],\n            autocolorscale = False,\n            marker = dict(\n                line = dict (\n                    color = 'rgb(180,180,180)',\n                    width = 0.5\n                ) ),\n            colorbar = dict(\n                autotick = False,\n                tickprefix = '',\n                title = 'Number of Visits'),\n          ) ]\n\n    layout = dict(\n        title = 'Couting Visits Per Country',\n        geo = dict(\n            showframe = False,\n            showcoastlines = True,\n            projection = dict(\n                type = 'Mercator'\n            )\n        )\n    )\n\n    figure = dict( data=data, layout=layout )\n    iplot(figure, validate=False, filename='map-countrys-count')\nplotmapvisit(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c482fa9ba39d129219963d3844d791c2638644dd"},"cell_type":"markdown","source":"**<font size=3>leak data set : vistis</font>**"},{"metadata":{"trusted":true,"_uuid":"789330efcf6994f795f8527c7f267e5cc893a5dc"},"cell_type":"code","source":"plotmapvisit(train_leak)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4458cdb0536815a8911b8327fbc30e7d8481afc5"},"cell_type":"markdown","source":"\n**<font size=3>whole data set :  revenues counts</font>**"},{"metadata":{"trusted":true,"_uuid":"f22b29e7b2808c01f4a838e11c1c8b22bbed03ba","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":" def plotmaprevenues(df_train):\n    # I will crete a variable of Revenues by country sum\n    sumRevMaps = df_train[df_train['totals.transactionRevenue'] > 0].groupby(\"geoNetwork.country\")[\"totals.transactionRevenue\"].count().to_frame().reset_index()\n    sumRevMaps.columns = [\"country\", \"count_sales\"] # renaming columns\n    sumRevMaps = sumRevMaps.reset_index().drop('index', axis=1) #reseting index and drop index column\n\n    data = [ dict(\n            type = 'choropleth',\n            locations = sumRevMaps['country'],\n            locationmode = 'country names',\n            z = sumRevMaps['count_sales'],\n            text = sumRevMaps['country'],\n            autocolorscale = False,\n            marker = dict(\n                line = dict (\n                    color = 'rgb(180,180,180)',\n                    width = 0.5\n                ) ),\n            colorbar = dict(\n                autotick = False,\n                tickprefix = '',\n                title = 'Count of Sales'),\n          ) ]\n\n    layout = dict(\n        title = 'Total Sales by Country',\n        geo = dict(\n            showframe = False,\n            showcoastlines = True,\n            projection = dict(\n                type = 'Mercator'\n            )\n        )\n    )\n\n    figure = dict( data=data, layout=layout )\n\n    iplot(figure, validate=False, filename='map-countrys-total')\nplotmaprevenues(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"681750e11be4cae355f3c15b2aacd177d92aa20a"},"cell_type":"markdown","source":"**<font size=3>leak data set :  revenues counts</font>**"},{"metadata":{"trusted":true,"_uuid":"6bc84192609f7e3cf2fa543cb3dbea2ab0ecaa27"},"cell_type":"code","source":"plotmaprevenues(train_leak)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"149dd76907ccf67b6ec3f4d3226f204f508c2e8c"},"cell_type":"markdown","source":"**<font size=3>It is my first EDA:))))))))))</font>**"},{"metadata":{"trusted":true,"_uuid":"40d510c53ff8eb9ca0516216b451ed676ee0d3ed"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}