{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#reading dataset\ndf_train = pd.read_csv(\"/kaggle/input/ga-customer-revenue-prediction/train.csv\")\ndf_train.head()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#convert json columns\n\nimport os\nimport json\nfrom pandas.io.json import json_normalize\n\ndef load_df(csv_path='/kaggle/input/ga-customer-revenue-prediction/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\ntrain_df = load_df('/kaggle/input/ga-customer-revenue-prediction/train.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#better description for dataset\n\nfrom scipy import stats\n\n\ndef DataDesc(df):\n    print(f\"Dataset Shape: {df.shape}\")\n    summary = pd.DataFrame(df.dtypes,columns=['dtypes'])\n    summary = summary.reset_index()\n    summary['Name'] = summary['index']\n    summary = summary[['Name','dtypes']]\n    summary['Missing'] = df.isnull().sum().values    \n    summary['Uniques'] = df.nunique().values\n    summary['First Value'] = df.loc[0].values\n    summary['Second Value'] = df.loc[1].values\n    summary['Third Value'] = df.loc[2].values\n\n    for name in summary['Name'].value_counts().index:\n        summary.loc[summary['Name'] == name, 'Entropy'] = round(stats.entropy(df[name].value_counts(normalize=True), base=2),2) \n    \n    return summary","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DataDesc(train_df)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.isnull().sum()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#imputation of null values and converting the columns values to int\n\ndef fill_na(df):   \n    df['totals.pageviews'].fillna(1, inplace=True)\n    df['totals.newVisits'].fillna(0, inplace=True)\n    df['totals.bounces'].fillna(0, inplace=True) \n    df[\"totals.transactionRevenue\"].fillna(0.0, inplace=True)\n    \n    # Changing datatypes from object to desired ones\n    df['totals.pageviews'] = df['totals.pageviews'].astype(int)\n    df['totals.newVisits'] = df['totals.newVisits'].astype(int)\n    df['totals.bounces'] = df['totals.bounces'].astype(int)\n    df[\"totals.transactionRevenue\"] = df[\"totals.transactionRevenue\"].astype(float)\n    \n    \n    df['trafficSource.isTrueDirect'].fillna(False, inplace=True) \n    df['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True) # filling boolean with True\n    df[train_df['geoNetwork.city'] == \"(not set)\"]['geoNetwork.city'] = np.nan\n    df['geoNetwork.city'].fillna(\"NaN\", inplace=True)\n    \n    return df\n\ndf = fill_na(train_df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DataDesc(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#these columns has very large amount of null values so its better to drop it\n\ndf.drop(columns=['trafficSource.adContent', 'trafficSource.adwordsClickInfo.adNetworkType', 'trafficSource.adwordsClickInfo.gclId',\n                'trafficSource.adwordsClickInfo.slot', 'trafficSource.adwordsClickInfo.page', 'trafficSource.referralPath',\n                 'trafficSource.keyword'], inplace = True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for col in df.columns:\n    if len(df[col].unique()) == 1:\n        df.drop(col,inplace=True,axis=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"DataDesc(df)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#converting to float\n\ndf['fullVisitorId'] = df['fullVisitorId'].astype(float)\n\ndf['sessionId'] = df['sessionId'].astype(float)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# This function is to extract date features\n\nfrom datetime import datetime\n\n\ndef date_process(df):\n    df[\"date\"] = pd.to_datetime(df[\"date\"], format=\"%Y%m%d\") # seting the column as pandas datetime\n    df[\"weekday\"] = df['date'].dt.weekday #extracting week day\n    df[\"day\"] = df['date'].dt.day # extracting day\n    df[\"month\"] = df['date'].dt.month # extracting day\n    df[\"year\"] = df['date'].dt.year # extracting day\n    df['visitHour'] = (df['visitStartTime'].apply(lambda x: str(datetime.fromtimestamp(x).hour))).astype(int)\n    \n    return df\ndf = date_process(df)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visalising the broweser column and revenue column\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\ndf.groupby('device.browser')['totals.transactionRevenue'].count().plot(kind='barh', figsize=(9,9))  \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising Operating system vs Revenue\n\n\ndf.groupby('device.operatingSystem')['totals.transactionRevenue'].count().plot(kind='barh', figsize=(5,5))  \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising ismobile vs Revenue\n\n\ndf.groupby('device.isMobile')['totals.transactionRevenue'].count().plot(kind='barh', figsize=(3,3))  \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising continent vs Revenue\n\n\ndf.groupby('geoNetwork.continent')['totals.transactionRevenue'].count().plot(kind='barh', figsize=(3,3))  \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#visualising subcontinent vs Revenue\n\ndf.groupby('geoNetwork.subContinent')['totals.transactionRevenue'].count().plot(kind='barh', figsize=(5,5))  \n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising geonetworkcity vs Revenue\n\n\ndf.groupby(['geoNetwork.country'])['totals.transactionRevenue'].count().plot()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising pageviews vs Revenue\n\n\ndf.groupby(['totals.pageviews'])['totals.transactionRevenue'].count().plot()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising date vs Revenue\n\n\ndf.groupby(['date'])['totals.transactionRevenue'].count().plot(figsize = (7,7))\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising channel grouping vs Revenue\n\ndf.groupby(['channelGrouping'])['totals.transactionRevenue'].count().plot(kind='barh')\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"##visualising day vs Revenue\n\n\ndf.groupby(['day'])['totals.transactionRevenue'].count().plot()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n#visualising weekday vs Revenue\n\ndf.groupby(['weekday'])['totals.transactionRevenue'].count().plot()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising month vs Revenue\n\ndf.groupby(['month'])['totals.transactionRevenue'].count().plot()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising visit hour vs Revenue\n\ndf.groupby(['visitHour'])['totals.transactionRevenue'].count().plot()\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#visualising the distribution of Revenue column... we can see outliers... we need to sort it out after encoding\n\nsns.set_color_codes()\nax = sns.distplot([df['totals.transactionRevenue']>0], color=\"y\", bins=2)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn import model_selection, preprocessing, metrics\nimport lightgbm as lgb\n\n\n# Impute 0 for missing target values\ndf[\"totals.transactionRevenue\"].fillna(0, inplace=True)\ntrain_y = df[\"totals.transactionRevenue\"].values\ntrain_id = df[\"fullVisitorId\"].values\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.columns","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# label encode the categorical variables \n\ncat_cols = ['channelGrouping','device.browser',\n       'device.operatingSystem', 'device.isMobile', 'device.deviceCategory',\n       'geoNetwork.continent', 'geoNetwork.subContinent', 'geoNetwork.country',\n       'geoNetwork.region', 'geoNetwork.metro', 'geoNetwork.city',\n       'geoNetwork.networkDomain','trafficSource.campaign', 'trafficSource.source',\n       'trafficSource.medium', 'trafficSource.isTrueDirect',\n       'trafficSource.adwordsClickInfo.isVideoAd',\n       'trafficSource.campaignCode']\n\nfor col in cat_cols:\n    print(col)\n    lbl = preprocessing.LabelEncoder()\n    lbl.fit(list(df[col].values.astype('str')))\n    df[col] = lbl.transform(list(df[col].values.astype('str')))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#converting these columns to float\nnum_cols = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \"visitStartTime\", 'totals.bounces',  'totals.newVisits']    \nfor col in num_cols:\n    df[col] = df[col].astype(float)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking datatypes of the columns\ndf.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}