{"cells":[{"metadata":{"trusted":true,"_uuid":"774bbe9e6761671b8084c29570a3858423e70aac"},"cell_type":"code","source":"import os\nimport time\nimport gc\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# data manipulation\nimport json\nfrom pandas.io.json import json_normalize\nimport numpy as np\nimport pandas as pd\n# plot\nimport matplotlib.pyplot as plt\nimport seaborn as sns\ncolor = sns.color_palette()\n# model\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f76101d303c68b3e2618846f21d01c7a7685561"},"cell_type":"code","source":"%%time\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded {os.path.basename(csv_path)}. Shape: {df.shape}\")\n    return df\ntrain = load_df('../input/train.csv')\ntest = load_df('../input/test.csv')\nsub = pd.read_csv('../input/sample_submission.csv')\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96c9fb538a0e17b7f7e2091e3e493e9e7d8eba51"},"cell_type":"code","source":"print(train.shape)\nprint(test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"512b75defc68008596f759f64146a3060e02a04b"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"34f5a6f08f50a789e15f7a839382fd1aebdc77f5"},"cell_type":"code","source":"set(train.columns).difference(set(test.columns))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af1fa34249bdc340e5ca690d79b3cf5a270e1ab0"},"cell_type":"code","source":"cons_col = [i for i in train.columns if train[i].nunique(dropna=False)==1]\ncons_col","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e3120eee947f43bbce51343ee5bec9b6e8ebd6c"},"cell_type":"code","source":"train = train.drop(cons_col + ['trafficSource.campaignCode'], axis=1)\ntest = test.drop(cons_col, axis=1)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4492bbf5ee2266b2be5e240c02e58b3a47169af4"},"cell_type":"code","source":"def find_missing(data):\n    # number of missing values\n    count_missing = data.isnull().sum().values\n    # total records\n    total = data.shape[0]\n    # percentage of missing\n    ratio_missing = count_missing/total\n    # return a dataframe to show: feature name, # of missing and % of missing\n    return pd.DataFrame(data={'missing_count':count_missing, 'missing_ratio':ratio_missing}, index=data.columns.values)\ntrain_missing = find_missing(train)\ntest_missing = find_missing(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed14a55db70c97961aa0d735d78914f1bf4a15f0"},"cell_type":"code","source":"train_missing.reset_index()[['index', 'missing_ratio']]\\\n    .merge(test_missing.reset_index()[['index', 'missing_ratio']], on='index', how='left')\\\n    .rename(columns={'index':'columns', 'missing_ratio_x':'train_missing_ratio', 'missing_ratio_y':'test_missing_ratio'})\\\n    .sort_values(['train_missing_ratio', 'test_missing_ratio'], ascending=False)\\\n    .query('train_missing_ratio>0')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2194707ebb71322ba37fce8550eed64c9f73cc13"},"cell_type":"code","source":"if test.fullVisitorId.nunique() == len(sub):\n    print('Till now, the number of fullVisitorId is equal to the rows in submission. Everything goes well!')\nelse:\n    print('Check it again')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc192c4e2a5a522890f62a3db02028ba23806160"},"cell_type":"code","source":"y = np.nan_to_num(np.array([float(i) for i in train['totals.transactionRevenue']]))\nprint('The ratio of customers with transaction revenue is', str((y != 0).mean()))  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ca40e04ebe56b8bf15bf20ebff13eb3f03122d7"},"cell_type":"code","source":"train[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float')\ntarget = np.log1p(train.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum())\nprint('The ratio of customers with transaction revenue is', str((target != 0).mean()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24a911c24b791009d7aeebced18a4a73b2cf3372"},"cell_type":"code","source":"train['totals.bounces'] = train['totals.bounces'].fillna('0')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50f3d21740210939fb6cb801137efd5024202ffc"},"cell_type":"code","source":"train['totals.newVisits'] = train['totals.newVisits'].fillna('0')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b94c6423976de8a4bafce207916c055b614d8f54"},"cell_type":"code","source":"train_idx = train.fullVisitorId\ntest_idx = test.fullVisitorId\ntrain[\"totals.transactionRevenue\"] = train[\"totals.transactionRevenue\"].astype('float').fillna(0)\ntrain_y = train[\"totals.transactionRevenue\"]\ntrain_target = np.log1p(train.groupby(\"fullVisitorId\")[\"totals.transactionRevenue\"].sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"104e69db9b3b51ab58bb3435b7c05cd2e326039d"},"cell_type":"code","source":"def browser_mapping(x):\n    browsers = ['chrome','safari','firefox','internet explorer','edge','opera','coc coc','maxthon','iron']\n    if x in browsers:\n        return x.lower()\n    elif  ('android' in x) or ('samsung' in x) or ('mini' in x) or ('iphone' in x) or ('in-app' in x) or ('playstation' in x):\n        return 'mobile browser'\n    elif  ('mozilla' in x) or ('chrome' in x) or ('blackberry' in x) or ('nokia' in x) or ('browser' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif  ('lunascape' in x) or ('netscape' in x) or ('blackberry' in x) or ('konqueror' in x) or ('puffin' in x) or ('amazon' in x):\n        return 'mobile browser'\n    elif '(not set)' in x:\n        return x\n    else:\n        return 'others'\n    \n    \ndef adcontents_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('placement' in x) | ('placememnt' in x):\n        return 'placement'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'ad' in x:\n        return 'ad'\n    else:\n        return 'others'\n    \ndef source_mapping(x):\n    if  ('google' in x):\n        return 'google'\n    elif  ('youtube' in x):\n        return 'youtube'\n    elif '(not set)' in x or 'nan' in x:\n        return x\n    elif 'yahoo' in x:\n        return 'yahoo'\n    elif 'facebook' in x:\n        return 'facebook'\n    elif 'reddit' in x:\n        return 'reddit'\n    elif 'bing' in x:\n        return 'bing'\n    elif 'quora' in x:\n        return 'quora'\n    elif 'outlook' in x:\n        return 'outlook'\n    elif 'linkedin' in x:\n        return 'linkedin'\n    elif 'pinterest' in x:\n        return 'pinterest'\n    elif 'ask' in x:\n        return 'ask'\n    elif 'siliconvalley' in x:\n        return 'siliconvalley'\n    elif 'lunametrics' in x:\n        return 'lunametrics'\n    elif 'amazon' in x:\n        return 'amazon'\n    elif 'mysearch' in x:\n        return 'mysearch'\n    elif 'qiita' in x:\n        return 'qiita'\n    elif 'messenger' in x:\n        return 'messenger'\n    elif 'twitter' in x:\n        return 'twitter'\n    elif 't.co' in x:\n        return 't.co'\n    elif 'vk.com' in x:\n        return 'vk.com'\n    elif 'search' in x:\n        return 'search'\n    elif 'edu' in x:\n        return 'edu'\n    elif 'mail' in x:\n        return 'mail'\n    elif 'ad' in x:\n        return 'ad'\n    elif 'golang' in x:\n        return 'golang'\n    elif 'direct' in x:\n        return 'direct'\n    elif 'dealspotr' in x:\n        return 'dealspotr'\n    elif 'sashihara' in x:\n        return 'sashihara'\n    elif 'phandroid' in x:\n        return 'phandroid'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'mdn' in x:\n        return 'mdn'\n    elif 'duckduckgo' in x:\n        return 'duckduckgo'\n    elif 'seroundtable' in x:\n        return 'seroundtable'\n    elif 'metrics' in x:\n        return 'metrics'\n    elif 'sogou' in x:\n        return 'sogou'\n    elif 'businessinsider' in x:\n        return 'businessinsider'\n    elif 'github' in x:\n        return 'github'\n    elif 'gophergala' in x:\n        return 'gophergala'\n    elif 'yandex' in x:\n        return 'yandex'\n    elif 'msn' in x:\n        return 'msn'\n    elif 'dfa' in x:\n        return 'dfa'\n    elif '(not set)' in x:\n        return '(not set)'\n    elif 'feedly' in x:\n        return 'feedly'\n    elif 'arstechnica' in x:\n        return 'arstechnica'\n    elif 'squishable' in x:\n        return 'squishable'\n    elif 'flipboard' in x:\n        return 'flipboard'\n    elif 't-online.de' in x:\n        return 't-online.de'\n    elif 'sm.cn' in x:\n        return 'sm.cn'\n    elif 'wow' in x:\n        return 'wow'\n    elif 'baidu' in x:\n        return 'baidu'\n    elif 'partners' in x:\n        return 'partners'\n    else:\n        return 'others'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"20fe40dfd8536ea4220a18e658a4a23dc21212c5"},"cell_type":"code","source":"train['device.browser'] = train['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.adContent'] = train['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntrain['trafficSource.source'] = train['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')\n\ntest['device.browser'] = test['device.browser'].map(lambda x:browser_mapping(str(x).lower())).astype('str')\ntest['trafficSource.adContent'] = test['trafficSource.adContent'].map(lambda x:adcontents_mapping(str(x).lower())).astype('str')\ntest['trafficSource.source'] = test['trafficSource.source'].map(lambda x:source_mapping(str(x).lower())).astype('str')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81b435c83e8880ad440afa6b6168ae1af3c23113"},"cell_type":"code","source":"train.drop(['fullVisitorId', 'sessionId', 'visitId'], axis = 1, inplace = True)\ntest.drop(['fullVisitorId', 'sessionId', 'visitId'], axis = 1, inplace = True)\nnum_col = [\"totals.hits\", \"totals.pageviews\", \"visitNumber\", \"visitStartTime\", 'totals.bounces',  'totals.newVisits']\nfor i in num_col:\n    train[i] = train[i].astype('float').fillna(0)\n    test[i] = test[i].astype('float').fillna(0)\ncat_col = [e for e in train.columns.tolist() if e not in num_col]\ncat_col.remove('date')\ncat_col.remove('totals.transactionRevenue')\nfor i in cat_col:\n    lab_en = LabelEncoder()\n    train[i] = train[i].fillna('not known')\n    test[i] = test[i].fillna('not known')\n    lab_en.fit(list(train[i].astype('str')) + list(test[i].astype('str')))\n    train[i] = lab_en.transform(list(train[i].astype('str')))\n    test[i] = lab_en.transform(test[i].astype('str'))\n    print('finish', i)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed5c18b3310ac65ad1f309b2906832d8229b4293"},"cell_type":"code","source":"y_train = np.log1p(train[\"totals.transactionRevenue\"])\nx_train = train.drop([\"totals.transactionRevenue\",'date'], axis=1)\nx_test = test.copy()\nx_test = x_test.drop('date',axis=1)\nprint(x_train.shape)\nprint(x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76fb9e25728d63da164f514f76ad79824b204d9f"},"cell_type":"code","source":"folds = KFold(n_splits=5,random_state=6)\noof_preds = np.zeros(x_train.shape[0])\nsub_preds = np.zeros(x_test.shape[0])\n\nstart = time.time()\nvalid_score = 0\nfor n_fold, (trn_idx, val_idx) in enumerate(folds.split(x_train, y_train)):\n    trn_x, trn_y = x_train.iloc[trn_idx], y_train[trn_idx]\n    val_x, val_y = x_train.iloc[val_idx], y_train[val_idx]    \n    \n    train_data = lgb.Dataset(data=trn_x, label=trn_y)\n    valid_data = lgb.Dataset(data=val_x, label=val_y)\n    \n    params = {\"objective\" : \"regression\", \"metric\" : \"rmse\", 'n_estimators':10000, 'early_stopping_rounds':100,\n              \"num_leaves\" : 60, \"learning_rate\" : 0.01, \"bagging_fraction\" : 0.9,\n              \"feature_fraction\" : 0.3, \"bagging_seed\" : 0}\n    \n    lgb_model = lgb.train(params, train_data, valid_sets=[train_data, valid_data], verbose_eval=1000) \n    \n    oof_preds[val_idx] = lgb_model.predict(val_x, num_iteration=lgb_model.best_iteration)\n    oof_preds[oof_preds<0] = 0\n    sub_pred = lgb_model.predict(x_test, num_iteration=lgb_model.best_iteration) / folds.n_splits\n    sub_pred[sub_pred<0] = 0 # should be greater or equal to 0\n    sub_preds += sub_pred\n    print('Fold %2d RMSE : %.6f' % (n_fold + 1, np.sqrt(mean_squared_error(val_y, oof_preds[val_idx]))))\n    valid_score += np.sqrt(mean_squared_error(val_y, oof_preds[val_idx]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bac52f6a53f3224ac1347172716588cd182f53bf"},"cell_type":"code","source":"print('Session-level CV-score:', str(round(valid_score/folds.n_splits,4)))\nprint(' ')\ntrain_pred = pd.DataFrame({\"fullVisitorId\":train_idx})\ntrain_pred[\"PredictedLogRevenue\"] = np.expm1(oof_preds)\ntrain_pred = train_pred.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\ntrain_pred.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\ntrain_pred[\"PredictedLogRevenue\"] = np.log1p(train_pred[\"PredictedLogRevenue\"])\ntrain_rmse = np.sqrt(mean_squared_error(train_target, train_pred['PredictedLogRevenue']))\nprint('User-level score:', str(round(train_rmse, 4)))\nprint(' ')\nend = time.time()\nprint('training time:', str(round((end - start)/60)), 'mins')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8292cc671c82fe345b5079b10ae62076766f8bcb"},"cell_type":"code","source":"test_pred = pd.DataFrame({\"fullVisitorId\":test_idx})\ntest_pred[\"PredictedLogRevenue\"] = np.expm1(sub_preds)\ntest_pred = test_pred.groupby(\"fullVisitorId\")[\"PredictedLogRevenue\"].sum().reset_index()\ntest_pred.columns = [\"fullVisitorId\", \"PredictedLogRevenue\"]\ntest_pred[\"PredictedLogRevenue\"] = np.log1p(test_pred[\"PredictedLogRevenue\"])\ntest_pred.to_csv(\"lgb_Fet_eng_model.csv\", index=False) # submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93fa7c0f37d9a02ad2c6e2fb7348b4f02f0bd089"},"cell_type":"code","source":"lgb.plot_importance(lgb_model, height=0.5, max_num_features=20, ignore_zero = False, figsize = (12,6), importance_type ='gain')\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}