{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport gc\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"436c88e430fa126e53351c2a0c5aeee93afe26c5"},"cell_type":"markdown","source":"**Reading JSON Normalized Data **\n> which i have saved separately because it takes quite alot of time to normalize it\n\n> If you read it without specifying 'str' type for ** fullVisitorId**  it will show different number of rows."},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#cust.to_csv('train2.csv')\ncust=pd.read_csv('../input/normalize/train2.csv',index_col=0,dtype={'fullVisitorId': 'str'}, nrows=None)\n#cust_test.to_csv('test2.csv')\ncust_test=pd.read_csv('../input/normalize/test2.csv',index_col=0,dtype={'fullVisitorId': 'str'}, nrows=None)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"605e228581dc3dbdebef9df5c6e239e3b17700a6"},"cell_type":"code","source":"cust.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3cb7e35ddc7b7ecb7535994eed52a3dd65863819"},"cell_type":"markdown","source":"**Violin Plots**\n> Features vs TrasactionRevenue\n\n> See what features makes a difference. (Helps in cleaning and feature selection)"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d7ff2cafde2da719421b246d61e20ec2412376d6"},"cell_type":"code","source":"import seaborn as sns\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\ndef plot_violin(auto_prices, cols, col_y = 'totals.transactionRevenue'):\n    for col in cols:\n        fig = plt.figure(figsize=(40,20))\n        sns.set_style(\"whitegrid\")\n        sns.violinplot(col, col_y, data=auto_prices)\n        plt.xlabel(col) # Set text for the x axis\n        plt.ylabel(col_y)# Set text for y axis\n        #plt.tick_params(labelsize=10)\n        fig.show()\n\n#dont visualize without transforming date and visitStartTime otherwise it will show irrelevant graph and takes quite a time. \nirrelevant=['totals.transactionRevenue','fullVisitorId','sessionId','visitId','date','visitStartTime']        \ncat_cols=[]\n\nfor col in cust:\n   if(col not in irrelevant):\n    cat_cols.append(col)\n\nplot_violin(cust, cat_cols)    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a76e7f94cef1e343dcab46b423d8670bdfa59ef"},"cell_type":"markdown","source":"**Cleaning**\n> *1 :* Instead of filling **isTrueDirect** with False, I have filled it with False where **trafficSource.source** is not equals to **'(direct)'**.\n        > filled it initialy with 'none' because it was not detecting nans.\n\n> *2 :*  Transforming **date** into different format.\n\n> *3 :* Converting **visitStartTime** into seconds.\n\n> *4 :* Extracting day, month, year, time and day of week.\n\n> *5 :* Transforming **totals.transactionRevenue** into its log and filling nans with 0.\n\n> *6 :* Filling nans with relevant values. \n(Note that i haven't fill nans with **(not provided)** or **not available in demo dataset** as maybe there is some difference between them or they contain some information in them if we concatenate them it will be lost.\n\n> *7 :* Removing **trafficSource.campaignCode** as it is not available in test data.\n\n> *8 :* Removing Unique values column.\n\n> *9 :* Lower Casing values and replacing space with underscore."},{"metadata":{"trusted":true,"_uuid":"6c7bfd26ba238d6b7f195cfd76cf2c6f504ea1b6"},"cell_type":"code","source":"#1\ncust['trafficSource.isTrueDirect'].fillna('none',inplace=True)\ncust_test['trafficSource.isTrueDirect'].fillna('none',inplace=True)\ncust['trafficSource.isTrueDirect']=[False if (i[1]['trafficSource.isTrueDirect']=='none' and i[1]['trafficSource.source']!='(direct)') else i[1]['trafficSource.isTrueDirect'] for i in cust.iterrows()]\ncust_test['trafficSource.isTrueDirect']=[False if (i[1]['trafficSource.isTrueDirect']=='none' and i[1]['trafficSource.source']!='(direct)') else i[1]['trafficSource.isTrueDirect'] for i in cust_test.iterrows()]\ncust['trafficSource.isTrueDirect']=[True if (i[1]['trafficSource.isTrueDirect']=='none') else i[1]['trafficSource.isTrueDirect'] for i in cust.iterrows()]\ncust_test['trafficSource.isTrueDirect']=[True if (i[1]['trafficSource.isTrueDirect']=='none') else i[1]['trafficSource.isTrueDirect'] for i in cust_test.iterrows()]\n\n#2\ncust['date_new']=pd.to_datetime(cust['date'], format='%Y%m%d')\ncust['date']=cust['date_new']\ncust.drop(['date_new'],axis=1,inplace=True)\n\ncust_test['date_new']=pd.to_datetime(cust_test['date'], format='%Y%m%d')\ncust_test['date']=cust_test['date_new']\ncust_test.drop(['date_new'],axis=1,inplace=True)\n\n#3\ncust['visitStartTime_new']=pd.to_datetime(cust['visitStartTime'], unit='s')\ncust['visitStartTime']=cust['visitStartTime_new']\ncust.drop(['visitStartTime_new'],axis=1,inplace=True)\n\ncust_test['visitStartTime_new']=pd.to_datetime(cust_test['visitStartTime'], unit='s')\ncust_test['visitStartTime']=cust_test['visitStartTime_new']\ncust_test.drop(['visitStartTime_new'],axis=1,inplace=True)\n\n#cust_test['trafficSource.isTrueDirect'].fillna(False,inplace=True)\n\n#****************************************************************************************************\n#4\ncust['day']=cust['date'].dt.day\ncust['DOW'] = cust['date'].dt.dayofweek\ncust['month']=cust['date'].dt.month\ncust['year']=cust['date'].dt.year\ncust['time']=cust['visitStartTime'].dt.hour\ncust.drop(['visitStartTime'],axis=1,inplace=True)\n\ncust_test['day']=cust_test['date'].dt.day\ncust_test['DOW'] = cust_test['date'].dt.dayofweek\ncust_test['month']=cust_test['date'].dt.month\ncust_test['year']=cust_test['date'].dt.year\ncust_test['time']=cust_test['visitStartTime'].dt.hour\ncust_test.drop(['visitStartTime'],axis=1,inplace=True)\n\n\n#5\ncust['totals.transactionRevenue'] = pd.to_numeric(cust['totals.transactionRevenue'])\ncust['totals.transactionRevenue']=cust['totals.transactionRevenue'].apply(np.log)\ncust['totals.transactionRevenue'].fillna(0,inplace=True)\n\n\n#***************************************************************************************\n#6\ncust['trafficSource.adContent'].fillna('NA',inplace=True)\ncust_test['trafficSource.adContent'].fillna('NA',inplace=True)\n\ncust['trafficSource.adwordsClickInfo.gclId'].fillna('other',inplace=True)\ncust_test['trafficSource.adwordsClickInfo.gclId'].fillna('other',inplace=True)\n\ncust['trafficSource.adwordsClickInfo.slot'].fillna('other',inplace=True)\ncust_test['trafficSource.adwordsClickInfo.slot'].fillna('other',inplace=True)\n\ncust['trafficSource.adwordsClickInfo.isVideoAd'].fillna('True',inplace=True)\ncust_test['trafficSource.adwordsClickInfo.isVideoAd'].fillna('True',inplace=True)\n\ncust['trafficSource.adwordsClickInfo.page'].fillna('oth',inplace=True)\ncust_test['trafficSource.adwordsClickInfo.page'].fillna('oth',inplace=True)\n\ncust['trafficSource.adwordsClickInfo.adNetworkType'].fillna('other',inplace=True)\ncust_test['trafficSource.adwordsClickInfo.adNetworkType'].fillna('other',inplace=True)\n\ncust['trafficSource.keyword'].fillna('other',inplace=True)\ncust_test['trafficSource.keyword'].fillna('other',inplace=True)\n\ncust['trafficSource.referralPath'].fillna('other',inplace=True)\ncust_test['trafficSource.referralPath'].fillna('other',inplace=True)\n\n#cust.sort_values(by=['totals.transactionRevenue'],inplace=True)\n#cust.drop_duplicates(subset = 'sessionId', keep = 'last', inplace = True)\n\ncust['totals.newVisits'].fillna(0,inplace=True)\ncust_test['totals.newVisits'].fillna(0,inplace=True)\n\ncust['totals.bounces'].fillna(0,inplace=True)\ncust_test['totals.bounces'].fillna(0,inplace=True)\n\n#7\ncust.drop(['trafficSource.campaignCode'],axis=1,inplace=True)\n\n#*************************************************************************************************\n#8\nprint(\"Removing Unique Value columns: \")\nfor col in cust:\n    if(len(cust[col].unique())==1):\n        print(col)\n        cust.drop([col],axis=1,inplace=True)\n        cust_test.drop([col],axis=1,inplace=True)\n\n#9        \nfor col in cust:\n    cust[col]=cust[col].apply(lambda x:str(x).lower())\n    cust[col]=cust[col].apply(lambda x: str(x).replace(\" \",\"_\"))\nfor col in cust_test:    \n    cust_test[col]=cust_test[col].apply(lambda x:str(x).lower())\n    cust_test[col]=cust_test[col].apply(lambda x: str(x).replace(\" \",\"_\"))\n\n        \n#cust['totals.pageviews'].fillna('none',inplace=True)\n#cust_test['totals.pageviews'].fillna('none',inplace=True)\n#cust['totals.pageviews']=[i[1]['totals.hits'] if i[1]['totals.pageviews']=='none' else i[1]['totals.pageviews'] for i in cust.iterrows()]\n#cust_test['totals.pageviews']=[i[1]['totals.hits'] if i[1]['totals.pageviews']=='none' else i[1]['totals.pageviews'] for i in cust_test.iterrows()]\n#cust['totals.pageviews'] = pd.to_numeric(cust['totals.pageviews'])\n#cust_test['totals.pageviews'] = pd.to_numeric(cust_test['totals.pageviews'])\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0fc1c32691f6cd4eecf68e1dbb37f536583725f2"},"cell_type":"markdown","source":"**Reducing Categorical Features classes**\n> Replacing features values with **transactionRevenue** 0 to 'other'.\n\n> Its not an efficient or fast code. Sorry for that."},{"metadata":{"trusted":true,"_uuid":"78b2b95014eaa856fbe53527d906ee082ff0bb2d","scrolled":true},"cell_type":"code","source":"#**************************************************************************************************\ncols=['device.browser','device.operatingSystem','trafficSource.source','trafficSource.adContent','trafficSource.keyword',\n      'trafficSource.campaign','trafficSource.adwordsClickInfo.page','geoNetwork.region','geoNetwork.metro', \n      'geoNetwork.country','geoNetwork.city']\n\ncust['totals.transactionRevenue']=cust['totals.transactionRevenue'].astype(float)\n\nfor col in cols:\n    print(\"Working on : \")\n    print(col)\n    grouped=cust[[col,'totals.transactionRevenue']].groupby([col]).sum()\n    temp=grouped[grouped['totals.transactionRevenue']==0]\n    temp=temp['totals.transactionRevenue']\n    temp=np.array(temp.keys())\n    cust[col].replace(temp,'other',inplace=True)\n\n    temp=grouped[grouped['totals.transactionRevenue']>0]\n    temp=temp['totals.transactionRevenue']\n    temp=np.array(temp.keys())\n    temp2=pd.DataFrame()\n    temp2['join']=[i if i not in temp else 'other' for i in cust_test[col]]\n    temp2=set(temp2['join'])\n    temp2.remove('other')\n    temp3=[]\n    for i in temp2:\n        temp3.append(i)\n\n    cust_test[col].replace(temp3,'other',inplace=True)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1945ede50488f08a8c4b8170dd1eecca4b55c4c8"},"cell_type":"markdown","source":"**Visualization After Cleaning**"},{"metadata":{"trusted":true,"_uuid":"4fa152c753e538e1382ac776be94cb05618b06db"},"cell_type":"code","source":"cat_cols = ['DOW','trafficSource.adContent','time','channelGrouping','day', 'year', 'month', 'visitNumber', \n            'device.browser','device.deviceCategory','device.isMobile','device.operatingSystem','geoNetwork.city',\n            'geoNetwork.continent','geoNetwork.country','geoNetwork.subContinent','geoNetwork.metro','geoNetwork.region',\n           'totals.hits','totals.pageviews','trafficSource.isTrueDirect','trafficSource.medium','trafficSource.source']\n\nplot_violin(cust, cat_cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27b2c044554f490d26b25413ee41402a8526fa9d"},"cell_type":"code","source":"cust.to_csv('train3.csv')\ncust_test.to_csv('test3.csv')\n#cust=pd.read_csv('../input/simple/train3.csv',index_col=0,dtype={'fullVisitorId': 'str'}, nrows=None)\n#cust_test=pd.read_csv('../input/simple/test3.csv',index_col=0,dtype={'fullVisitorId': 'str'}, nrows=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}