{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom pandas import Series,DataFrame\nimport numpy as np\nimport json\nimport gc\nfrom datetime import datetime\nfrom pandas.io.json import json_normalize\nimport seaborn as sns\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Y_pred_train\ntrain_data = pd.read_csv(\"/kaggle/input/ga-customer-revenue-prediction/train.csv\",dtype={'fullVisitorId': 'str'},low_memory=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=train_data.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Creating array of column names that can be dropped from dataset\ndrop_columns=['date','fullVisitorId','sessionId','socialEngagementType','visitId','visitStartTime']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Extracting day of the week from dataset and creating new column\ndf['dayofweek']=pd.to_datetime(df.date, format='%Y%m%d', errors='ignore').dt.weekday","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#creating method to convert JSON columns to columns\ndef json_flatten1 (data,json_columns):\n    try:\n        for column in json_columns:\n            column_to_df= json_normalize([json.loads(x) for x in data[column]])\n            data=column_to_df\n        return data\n    \n    except Exception as e:\n            raise Exception()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting device JSON column to columns\ndata_dev=json_flatten1(df,['device'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dropping the columns which have unique values =1\nfor col in data_dev.columns:\n    if data_dev[col].nunique()==1:\n        data_dev=data_dev.drop(col,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_dev.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking for is null values\ndata_dev.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#creating method for marking as \"Others\" whose value_counts less than 100\ndef create_others(df,clm,leng):\n    s = df[clm].value_counts()\n    df[clm] = np.where(df[clm].isin(s.index[s < leng]), 'Other', df[clm])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Convertign values as others for \"borwser\" and \"operating system\" whose value_counts less tha 100\n#replacing isMobile boolen columns with 1 and o\ncreate_others(data_dev,'browser',100)\ncreate_others(data_dev,'operatingSystem',100)\ndata_dev['isMobile'].replace({True:1,False:0},inplace=True)\n#device cat fine","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting GeoNetwork JSON column to columns\ndata_geo=json_flatten1(df,['geoNetwork'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dropping the columns which have unique values =1\nfor col in data_geo.columns:\n    if data_geo[col].nunique()==1:\n        data_geo=data_geo.drop(col,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking for null values\ndata_geo.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_geo.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Repalcing the \"(not set) and (not available in demo dataset)\" with 0\ndata_geo.region.replace({'(not set)':0},inplace=True)\ndata_geo.region.replace({'not available in demo dataset':0},inplace=True)\n\ndata_geo.metro.replace({'(not set)':0},inplace=True)\ndata_geo.metro.replace({'not available in demo dataset':0},inplace=True)\n\ndata_geo.city.replace({'(not set)':0},inplace=True)\ndata_geo.city.replace({'not available in demo dataset':0},inplace=True)\n\ndata_geo.networkDomain.replace({'(not set)':0},inplace=True)\ndata_geo.networkDomain.replace({'not available in demo dataset':0},inplace=True)\n\n#creating others for value_counts less than 100\ncreate_others(data_geo,'region',100)\ncreate_others(data_geo,'metro',100)\ncreate_others(data_geo,'city',100)\ncreate_others(data_geo,'networkDomain',100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_geo.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting totals JSON column to columns\ndata_tot=json_flatten1(df,['totals'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_tot.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#checking for null values\ndata_tot.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#filling null values witj 0 or 1\ndata_tot.newVisits.fillna(value=1,inplace=True)\ndata_tot.pageviews.fillna(value=1,inplace=True)\ndata_tot.bounces.fillna(value=0,inplace=True)\ndata_tot.transactionRevenue.fillna(value=0,inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#converting trafficSource JSON column to columns\ndata_trf=json_flatten1(df,['trafficSource'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_trf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dropping the columns which have unique values =1\nfor col in data_trf.columns:\n    if data_trf[col].nunique()==1:\n        data_trf=data_trf.drop(col,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Checking for null values\ndata_trf.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Repalcing the \"(not set) and (not available in demo dataset)\" with 0\n#creating others for value_counts less than 100\ndata_trf.campaign.replace({'(not set)':0},inplace=True)\ncreate_others(data_trf,'campaign',100)\ncreate_others(data_trf,'source',100)\ndata_trf.medium.replace({'(not set)':0},inplace=True)\ndata_trf.medium.replace({'(none)':0},inplace=True)\ndata_trf.keyword.fillna(value=0,inplace=True)\ndata_trf.keyword.replace({'(not provided)':0},inplace=True)\ndata_trf.keyword[data_trf.keyword != 0]=1\n\ndata_trf.referralPath.fillna(value=0,inplace=True)\ndata_trf.referralPath[data_trf.referralPath != 0]=1\n\ndata_trf['adwordsClickInfo.page'].fillna(value=0,inplace=True)\n\ndata_trf['adwordsClickInfo.slot'].fillna(value=0,inplace=True)\ndata_trf['adwordsClickInfo.slot'][data_trf['adwordsClickInfo.slot'] != 0]=1\n\ndata_trf['adwordsClickInfo.gclId'].fillna(value=0,inplace=True)\ndata_trf['adwordsClickInfo.gclId'][data_trf['adwordsClickInfo.gclId'] != 0]=1\n\ndata_trf['adwordsClickInfo.adNetworkType'].fillna(value=0,inplace=True)\ndata_trf['adwordsClickInfo.adNetworkType'][data_trf['adwordsClickInfo.adNetworkType'] != 0]=1\n\ndata_trf['adContent'].fillna(value=0,inplace=True)\ndata_trf['adContent'][data_trf['adContent'] != 0]=1\n\n\n#create_others(data_trf,'referralPath',200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#merging all the JSON columns to dataset\ndf= df.merge(data_dev, right_index=True, left_index=True)\ndf= df.merge(data_geo, right_index=True, left_index=True)\ndf= df.merge(data_tot, right_index=True, left_index=True)\ndf= df.merge(data_trf, right_index=True, left_index=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dropping origin JSON columns\ndf=df.drop(columns=['device','geoNetwork','totals','trafficSource'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Dropping unwanted columns\ndf=df.drop(columns=drop_columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1=df.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df1.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2=df1.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#df2=df2.drop(columns=['adwordsClickInfo.page','adwordsClickInfo.slot','adwordsClickInfo.gclId','adwordsClickInfo.adNetworkType','adContent'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Checking for Cat values for label encoding\ncategorical_features = [\n    _f for _f in df2.columns\n    if(df1[_f].dtype == 'object')\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#label encoder initialising\nfrom sklearn.preprocessing import LabelEncoder\nlbl = LabelEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for col in categorical_features:\n    lbl.fit(list(df2[col].values.astype('str')) + list(df2[col].values.astype('str')))\n    df2[col] = lbl.transform(list(df2[col].values.astype('str')))\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#random forest\nfrom sklearn.ensemble import RandomForestRegressor\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Y=np.log1p(df2.transactionRevenue)\nX=df2.drop(columns=['transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_test_split\nRF=RandomForestRegressor()\nX_train,X_test,Y_train,Y_test=train_test_split(X,Y,test_size=0.2,random_state=0)\nRF.fit(X,Y)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#predicting for test and train\nY_pred=RF.predict(X_test)\nY_pred_train=RF.predict(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import r2_score\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.metrics import mean_squared_log_error","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('test set'  ,r2_score(Y_test,Y_pred))\nprint('train set '  ,r2_score(Y_train,Y_pred_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('test set'  ,mean_squared_error(Y_test,Y_pred))\nprint('train set '  ,mean_squared_error(Y_train,Y_pred_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('test set' , np.sqrt(mean_squared_error(Y_test,Y_pred)))\nprint('train set '  ,np.sqrt(mean_squared_error(Y_train,Y_pred_train)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pickle\nfile=open('C:\\Lingesh_Vijay\\Learning\\ineuron\\Internship\\GoogleAnalytics.pkl','wb')\npickle.dump(RF,file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(file.name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}