{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport json\nfrom pandas.io.json import json_normalize\nimport time\nfrom datetime import datetime\nimport gc\nimport psutil\nfrom sklearn.preprocessing import LabelEncoder\n\ncolumns_to_parse = ['device','geoNetwork','totals','trafficSource']\n\ndef parse_dataframe(path):\n    #path = '/Users/abhayranjan/kaggle/google/all/train.csv'\n    data_df= pd.read_csv(path,converters={column: json.loads for column in columns_to_parse},\n                                dtype={'fullVisitorId':str})\n    \n    #parse the json type columns\n    for col in columns_to_parse:\n        json_col_df = json_normalize(data_df[col])\n        json_col_df.columns = [f\"{col}_{sub_col}\" for sub_col in json_col_df.columns]\n        # we drop the object column processed and we add the column created from json fields.\n        data_df = data_df.drop(col,axis=1).merge(json_col_df,right_index=True,left_index=True)\n        \n    return data_df \ndef process_datetime(data_df):\n    data_df['date'] = data_df['date'].astype(str)\n    data_df['date'] = data_df['date'].apply(lambda x:x[:4]+\"-\"+x[4:6]+\"-\"+x[6:])\n    data_df['date'] = pd.to_datetime(data_df['date'])\n    data_df['year'] = data_df['date'].dt.year\n    data_df['month'] = data_df['date'].dt.month\n    data_df['day'] = data_df['date'].dt.day\n    data_df['weekday'] =data_df['date'].dt.weekday\n    data_df['weekofyear'] =data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    return data_df\ndef process_format(data_df):\n    print(\"Inside process_format function\")\n    for col in ['visitNumber','totals_hits','totals_pageviews']:\n        data_df[col] = data_df[col].astype(float)\n            \n    data_df['trafficSource_adwordsClickInfo.isVideoAd'].fillna(True,inplace=True)\n    data_df['trafficSource_isTrueDirect'].fillna('False',inplace=True)\n    return data_df\ndef process_device(data_df):\n    print(\"process device\")\n    data_df['browser_category'] = data_df['device_browser'] + \"_\"+data_df['device_deviceCategory']\n    data_df['browser_os'] = data_df['device_browser'] + \"_\"+data_df['device_operatingSystem']\n    return data_df\ndef process_totals(data_df):\n    print(\"process totals..\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals_hits'] = np.log1p(data_df['totals_hits'])\n    data_df['totals_pageviews'] = np.log1p(data_df['totals_pageviews']).fillna(0)\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals_hits'].transform('var')\n    data_df['mean_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('mean')\n    data_df['sum_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('sum')\n    data_df['max_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('max')\n    data_df['min_pageviews_per_day'] = data_df.groupby(['day'])['totals_pageviews'].transform('min')\n    return data_df\ndef process_geo_network(data_df):\n    print(\"process geo network...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby(['geoNetwork_networkDomain'])['totals_pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby(['geoNetwork_networkDomain'])['totals_pageviews'].transform('count') \n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby(['geoNetwork_networkDomain'])['totals_pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby(['geoNetwork_networkDomain'])['totals_hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby(['geoNetwork_networkDomain'])['totals_hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby(['geoNetwork_networkDomain'])['totals_hits'].transform('mean')\n    return data_df                                                  \npd.set_option('display.max.columns',None)\ndef process_traffic_source(data_df):\n    print(\"process traffic source....\")\n    data_df['source_country'] = data_df['trafficSource_source']+\"_\" +data_df['geoNetwork_country']\n    data_df['campaign_medium'] = data_df['trafficSource_campaign']+\"_\" +data_df['trafficSource_medium']\n    data_df['medium_hits_mean'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('mean')\n    data_df['medium_hits_min'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('min')\n    data_df['medium_hits_max'] =data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('max')\n    data_df['medium_hits_sum'] = data_df.groupby(['trafficSource_medium'])['totals_hits'].transform('sum')\n    return data_df\ng_train_df = parse_dataframe('../input/train.csv')\ng_train_df = process_datetime(g_train_df)\ng_test_df = parse_dataframe('../input/test.csv')\ng_test_df = process_datetime(g_test_df) \n\ncols_to_drop = [col for col in g_train_df.columns if g_train_df[col].nunique(dropna=False) == 1]\ng_train_df.drop(cols_to_drop, axis=1, inplace=True)\ng_test_df.drop([col for col in cols_to_drop if col in g_test_df.columns], axis=1, inplace=True)\n\n###only one not null value\ng_train_df.drop(['trafficSource_campaignCode'], axis=1, inplace=True)\n\n###converting columns format\ng_train_df['totals_transactionRevenue'] = g_train_df['totals_transactionRevenue'].astype(float)\ng_train_df['totals_transactionRevenue'] = g_train_df['totals_transactionRevenue'].fillna(0)\n\ng_train_df = process_format(g_train_df)\ng_tain_df = process_device(g_train_df)\ng_train_df = process_totals(g_train_df)\ng_train_df = process_geo_network(g_train_df)\ng_train_df = process_traffic_source(g_train_df)   \n\ng_test_df = process_format(g_test_df)\ng_test_df = process_device(g_test_df)\ng_test_df = process_totals(g_test_df)\ng_test_df = process_geo_network(g_test_df)\ng_test_df = process_traffic_source(g_test_df)   \n\n# Numeric Columns\n\nnum_cols = ['month_unique_user_count','day_unique_user_count','weekday_unique_user_count','visitNumber','total_hits','total_pageviews','mean_hits_per_day','sum_hits_per_day','max_hits_per_day','min_hits_per_day','var_hits_per_day',\n           'mean_pageviews_per_day','sum_pageviews_per_day','max_pageviews_per_day','min_pageviews_per_day',\n           'sum_pageviews_per_network_domain','count_pageviews_per_network_domain','mean_pageviews_per_network_domain','sum_hits_per_network_domain',\n            'count_hits_per_network_domain','mean_hits_per_network_domain','medium_hits_mean',\n           'medium_hits_min','medium_hits_max','medium_hits_sum']\n\nnot_used_cols = ['visitNumber','date','fullVisitorId','sessionId','visitId','visitStartTime',\n                'totals_transactionRevenue','trafficSource_referralPath']\n\ncat_cols = [col for col in g_train_df.columns if col not in num_cols and col not in not_used_cols]\nprint(cat_cols)\n\nmerged_df = pd.concat([g_train_df,g_test_df])\nohe_columns = []\nfor i in cat_cols:\n    if len(set(merged_df[i].values))<100:\n        ohe_columns.append(i)\n            \nprint('ohe_cols:',ohe_columns)\n#print(len(ohe_columns))\ntrn_shape=g_train_df.shape[0]\nmerged_df = pd.get_dummies(merged_df,columns=ohe_columns)\ng_train_df = merged_df[:trn_shape]\ng_test_df  = merged_df[trn_shape:]\ng_train_df = g_train_df.loc[:,~g_train_df.columns.duplicated()]\ng_test_df = g_test_df.loc[:,~g_test_df.columns.duplicated()]\nfrom sklearn import preprocessing\nfrom sklearn.preprocessing import LabelEncoder\nfor col in cat_cols:\n    if col in ohe_columns:\n        continue\n    lbl = LabelEncoder()\n    lbl.fit(list(g_train_df[col].values.astype('str')) + list(g_test_df[col].values.astype('str')))\n    g_train_df[col]= lbl.transform(list(g_train_df[col].values.astype('str')))\n    g_test_df[col]= lbl.transform(list(g_test_df[col].values.astype('str')))      \n    \nX = g_train_df.drop(not_used_cols,axis=1)\ny = g_train_df['totals_transactionRevenue']\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import KFold,StratifiedKFold\nfrom sklearn import model_selection,preprocessing,metrics\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# Stratified Sampling\nFOLDs= StratifiedKFold(n_splits=5,shuffle=True,random_state=5)\noof_lgb = np.zeros(len(g_train_df))\npredictions_lgb = np.zeros(len(g_test_df))\n#print(list(X.columns))                          \nfeatures_lgb = list(X.columns)\nfeature_importance_df_lgb = pd.DataFrame()\n                           \nfor fold_, (trn_idx,val_idx) in enumerate(FOLDs.split(X,y_categorized)):\n    trn_data = lgb.Dataset(X.iloc[trn_idx],label=y_log.iloc[trn_idx])\n    val_data = lgb.Dataset(X.iloc[val_idx],label=y_log.iloc[val_idx])\n    num_round=2000\n    clf = lgb.train(lgb_params1,trn_data,num_round,valid_sets=[trn_data,val_data],verbose_eval=1000,early_stopping_rounds=100)\n    oof_lgb[val_idx] = clf.predict(X.iloc[val_idx],num_iteration=clf.best_iteration)\n    fold_importance_df_lgb = pd.DataFrame()\n    fold_importance_df_lgb[\"feature\"] = features_lgb\n    fold_importance_df_lgb[\"importance\"] = clf.feature_importance()\n    fold_importance_df_lgb[\"fold\"] = fold_ + 1\n    feature_importance_df_lgb = pd.concat([feature_importance_df_lgb,fold_importance_df_lgb],axis=0)\n    predictions_lgb += clf.predict(X_test,num_iteration=clf.best_iteration) / FOLDs.n_splits\n    \n    \ncols = feature_importance_df_lgb[[\"feature\",\"importance\"]].groupby(\"feature\").mean().sort_values(by=\"importance\",ascending=False)[:50].index\nbest_features_lgb = feature_importance_df_lgb.loc[feature_importance_df_lgb.feature.isin(cols)]\nplt.figure(figsize=(14,10))\nsns.barplot(x=\"importance\",y=\"feature\",data=best_features_lgb.sort_values(by=\"importance\",ascending=False))\nplt.title('LightGBM features( avg over folds)')\nplt.tight_layout()\nplt.savefig('lgbm_importancers.png')\nx = []\nfor i in oof_lgb:\n    if i<0:\n        x.append(0.0)\n    else:\n        x.append(i)\ncv_lgb = mean_squared_error(x,y_log)**0.5\ncv_lgb = str(cv_lgb)\ncv_lgb = cv_lgb[:10]\npd.DataFrame({'preds':x}).to_csv('lgb_oof_'+cv_lgb + '.csv',index=False)\nprint(\"CV_LGB:\",cv_lgb)\n\nsub_df = g_test_df[['fullVisitorId']].copy()\npredictions_lgb[predictions_lgb<0] = 0\nsub_df['PredictedLogRevenue'] = np.expm1(predictions_lgb)\nsub_df = sub_df.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\nsub_df.columns = ['fullVisitorId',\"PredictedLogRevenue\"]\n\nsub_df[\"PredictedLogRevenue\"] = np.log1p(sub_df[\"PredictedLogRevenue\"])\nsub_df.to_csv(\"submission.csv\",index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}