{"cells":[{"metadata":{"trusted":true,"_uuid":"6f0dbdb74e6f5fd7352d9cfb0040e126ebe67d38"},"cell_type":"code","source":"import os\nimport sys\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport json\nfrom pandas.io.json import json_normalize\nimport lightgbm as lgbm\nimport time\n\n%matplotlib inline\n%load_ext autoreload\n%autoreload 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57d237b3de76a9932e667b39f90591c6fdbd7792"},"cell_type":"code","source":"#Let's define some utilities\n\ndef load_df(csv_path='../input/train.csv', nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    print(f\"read from file '{csv_path}'...\")\n    df = pd.read_csv(csv_path,\n                     converters={\n                         column: json.loads for column in JSON_COLUMNS},\n                     dtype={'fullVisitorId': 'str'},  # Important!!\n                     nrows=nrows)\n    print(\"convert columns from json format to plain text...\")\n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [\n            f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(\n            column_as_df, right_index=True, left_index=True)\n    print(f\"Loaded data from '{os.path.basename(csv_path)}'. Shape: {df.shape}\")\n    return df\n\n\ndef count_unique(df):\n    for column in df.columns:\n            print(column, len(set(df[column])))\n            \ndef get_list_of_dummies(df):\n    result = []\n    for column in df.columns:\n        if len(set(df[column])) == 1:\n            result.append(column)\n            \n    return result\n\n\ndef get_list_of_values_for_columns_with_nulls(df):\n    result = {}\n    for column in df.columns:\n        s = set(df[column])\n        if \"null\" in s:\n            if len(s) < 10:\n                print(column, s)\n            else:\n                print(column, \" is \", len(s))\n            result[column] = s\n            \n        \n            \n    return result\n\ndef check_values_of_cat_features(train, test):\n    result = {}\n    for column in train.columns:\n        assert train[column].dtype == test[column].dtype\n        if train[column].dtype != np.object:\n            continue\n        train_set = set(train[column])\n        test_set = set(test[column])\n        result[column] = test_set - train_set\n        if not (test_set <= train_set):\n            \n            print(\"For column\", column)\n#             diff = test_set - train_set\n#             if len(diff) < 100:\n#                 print(diff)\n\n    return result\n\ndef transform_replace_value(df, mapping):\n    for column in df.columns:\n        if column in mapping:\n            cur = mapping[column]\n            \n            if isinstance(cur, dict):\n                df[column] = df[column].apply(cur['func'])\n                df[column] = df[column].astype(cur['dtype'])\n            else:\n                df[column] = df[column].apply(cur)\n            \n    \n    return df\n\ndef drop_rare_categories(train, test, min_freq, min_ratio):\n    feature_values = {}\n    columns = train.columns\n    for column in columns:\n        assert train[column].dtype == test[column].dtype\n        if train[column].dtype != np.object:\n            continue\n#         print(\"Processing \", column)\n    \n        values, counts = np.unique(train[column], return_counts=True)\n\n        argsort = (-counts).argsort()\n        counts = counts[argsort]\n        values = values[argsort]\n\n        if len(values) < min_freq:\n            continue\n        \n        values_to_spare = set(values[counts >= min_ratio * counts[0]])\n        \n        train[column] = train[column].apply(lambda v: v if v in values_to_spare else \"__rare__\")\n        test[column] = test[column].apply(lambda v: v if v in values_to_spare else \"__rare__\")\n        \n        feature_values[column] = values_to_spare | set(['__rare__'])\n        \n    \n    return train, test, feature_values\n        \n        \ndef print_value_counts(train):\n    for column in train.columns:\n        if train[column].dtype != np.object:\n            continue\n        print(column, len(set(train[column])))\n    \n    \n\ndef transform_add_is_present_flag(series, null_replacement, res_type=None):\n    if res_type is None:\n        res_type = series.dtype\n    is_present = (series != \"null\").astype(np.int64)\n    series = series.apply(lambda x: x if x != \"null\" else null_replacement).astype(res_type)\n    \n    return series, is_present\n\n\n\ndef plot_countplot(df, column):\n    plt.figure(figsize=(15, 15))\n    sns.countplot(data=df, y=column, order=df[column].value_counts().index, orient='h')\n    \ndef plot_countplot_series(series):\n    plt.figure(figsize=(15, 15))\n    sns.countplot(y=series, order=series.value_counts().index, orient='h')\n    \n\n    \nfrom sklearn.preprocessing import OneHotEncoder\ndef transform_categories(train, test):\n    transformers = {}\n    columns = train.columns\n    values_train = []\n    values_test = []\n    \n    cat_columns = []\n    for column in columns:\n        assert train[column].dtype == test[column].dtype\n        if train[column].dtype != np.object:\n            continue\n        cat_columns.append(column)\n#         print(\"Processing \", column)\n        \n        transformer = OneHotEncoder(handle_unknown='ignore', sparse=False, dtype=np.float32)\n        cur_train = train[column].values.reshape(-1, 1)\n        cur_test = test[column].values.reshape(-1, 1)\n        train = train.drop(column, axis=1)\n        test = test.drop(column, axis=1)\n        values_train.append(transformer.fit_transform(cur_train))\n        values_test.append(transformer.transform(cur_test))\n        \n        transformers[column] = transformer\n        \n    \n    feature_names = list(map(lambda x: \"num:\" + x, train.columns))\n    back_feature_names = {}\n    \n    for name in cat_columns:  \n        for feature in transformers[name].categories_[0]:\n            current_name = name + \":cat:\" + feature\n            back_feature_names[current_name] = len(feature_names)\n            feature_names.append(current_name)\n            \n            \n    \n    values_train = [train.values.astype(np.float32)] + values_train\n    values_test = [test.values.astype(np.float32)] + values_test\n    \n    x_train = np.concatenate(values_train, axis=1)\n    x_test = np.concatenate(values_test, axis=1)\n    \n    return x_train, x_test, feature_names, back_feature_names\n\n\n\ndef metric(true, pred):\n    assert set(true.keys()) == set(pred.keys())\n    \n    diff = []\n    for key in true.keys():\n        diff.append(true[key] - pred[key])\n    diff = np.array(diff)\n#     print(diff)\n    result = (sum(diff**2)/len(diff))**0.5\n    return result\n\n\nfrom collections import defaultdict\ndef make_prediction(users, predictions):\n    assert len(users) == len(predictions), (len(users), len(predictions))\n    result = defaultdict(list)\n    for user_id, pred in zip(users, predictions):\n        result[user_id].append(pred)\n    \n    for user_id in result:\n        result[user_id] = np.log(np.sum(result[user_id]) + 1)\n        \n    return result\n\n\ndef predict(X, cls, reg):\n    ind = cls.predict(X).astype(np.bool)\n    \n    result = np.zeros(shape=(len(X), ))\n    \n    \n    pr = reg.predict(X[ind])\n    result[ind] = np.maximum(np.exp(pr) - 1, 0.0)\n    \n    return result\n\n\ndef predict_reg(X, reg):\n    \n    result = np.maximum(np.exp(reg.predict(X)) - 1, 0.0)\n    \n    return result\n\n\ndef make_result_df(test_pred):\n    return pd.DataFrame(list(test_pred.items()), columns=['fullVisitorId','PredictedLogRevenue'])\n\ndef write_df(df, file):\n    df.sort_values(by='fullVisitorId').to_csv(file, index=False, header=True)\n    \n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f26b01e5b4afb39f006a0600e359723f0e61fe61"},"cell_type":"code","source":"train = load_df(\"../input/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f589e651604078f030cd8d0e7cbc2fc52a54fbeb"},"cell_type":"code","source":"test = load_df(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2626f052d56398054e1bf19831a3bd065d684460"},"cell_type":"code","source":"#define the data preprocessing pipeline:\nimport datetime\nfrom datetime import datetime\nimport time\nimport copy\n\n\n#used: https://www.kaggle.com/fabiendaniel/lgbm-starter \n#used: https://www.kaggle.com/ashishpatel26/updated-bayesian-lgbm-xgb-cat-fe-kfold-cv\n#used: https://www.kaggle.com/ashishpatel26/1-43-plb-feature-engineering-best-model-combined\n\n\ndef null_to_0(value):\n    if value != \"null\":\n        return value\n    return 0\n\ndef replace_value(value, mapping):\n    if value in mapping:\n        return mapping[value]\n    return value\n\n\nto_int = {\"func\":null_to_0, \"dtype\":np.int64}\nmapping = {\n    \"totals.bounces\":to_int,\n    \"totals.newVisits\":to_int,\n    \"trafficSource.adwordsClickInfo.isVideoAd\":to_int,\n    \"trafficSource.isTrueDirect\":to_int,\n    \"totals.hits\":to_int,\n    \"trafficSource.adwordsClickInfo.adNetworkType\":lambda value: replace_value(value, {\"Content\":\"null\"}),\n    \"trafficSource.adwordsClickInfo.slot\":lambda value: replace_value(value, {'Google Display Network':\"null\"})\n}\n\n\ndef extract_presence_flags(df):\n    name = 'totals.pageviews'\n    tmp = transform_add_is_present_flag(df[name], 0, res_type=np.int64)\n    df[name] = tmp[0]\n    df[name + '.is_present'] = tmp[1]\n    \n    return df\n\ndef drop_ids(df):\n    columns = ['fullVisitorId', 'sessionId', 'visitId']\n    ids = df[columns]\n    return df.drop(columns, axis=1), ids\n\ndef factorize(train, test, params={}):\n    columns = filter(lambda x: train[x].dtype == np.object, train.columns)\n    indexers = {}\n    for column in columns:\n        assert train[column].dtype == test[column].dtype\n        train[column], indexer = pd.factorize(train[column])\n        test[column] = indexer.get_indexer(test[column])\n        indexers[column] = indexer\n    \n    return train, test, indexers\n\ndef process_date_time(data_df):\n    print(\"process date time ...\")\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    data_df['date'] = data_df['date'].astype(str)\n    return data_df\n\n# def process_format(data_df):\n#     print(\"process format ...\")\n#     for col in ['visitNumber', 'totals.hits', 'totals.pageviews']:\n#         data_df[col] = data_df[col].astype(float)\n#     data_df['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n#     data_df['trafficSource.isTrueDirect'].fillna(False, inplace=True)\n#     return data_df\n    \ndef process_device(data_df):\n    print(\"process device ...\")\n    data_df['browser_category'] = data_df['device.browser'] + '_' + data_df['device.deviceCategory']\n    data_df['browser_operatingSystem'] = data_df['device.browser'] + '_' + data_df['device.operatingSystem']\n    data_df['source_country'] = data_df['trafficSource.source'] + '_' + data_df['geoNetwork.country']\n    return data_df\n\ndef process_totals(data_df):\n    print(\"process totals ...\")\n    data_df['visitNumber'] = np.log1p(data_df['visitNumber'])\n    data_df['totals.hits'] = np.log1p(data_df['totals.hits'])\n    data_df['totals.pageviews'] = np.log1p(data_df['totals.pageviews'].fillna(0))\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('var')\n    return data_df\n\ndef process_geo_network(data_df):\n    print(\"process geo network ...\")\n    data_df['sum_pageviews_per_network_domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('sum')\n    data_df['count_pageviews_per_network_domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('count')\n    data_df['mean_pageviews_per_network_domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('mean')\n    data_df['sum_hits_per_network_domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('sum')\n    data_df['count_hits_per_network_domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('count')\n    data_df['mean_hits_per_network_domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('mean')\n    return data_df\n\n\n\ndef add_features(df, params={}):\n    df = process_date_time(df)\n    df = process_device(df)\n    df = process_totals(df)\n    df = process_geo_network(df)\n\n\n    \n    return df\n\n\n\ndef transform_dataset(train, test, params):\n    \n    train = copy.deepcopy(train)\n    test = copy.deepcopy(test)\n    \n    feature_info = {\"params\":copy.deepcopy(params)}\n    train = train.fillna(\"null\")\n    test = test.fillna(\"null\")\n    columns_to_drop_test = get_list_of_dummies(train)\n    if 'trafficSource.campaignCode' in test.columns:\n        columns_to_drop_test += ['trafficSource.campaignCode']\n    else:\n        columns_to_drop_train = columns_to_drop_test + ['trafficSource.campaignCode']\n        \n    test = test.drop(columns_to_drop_test, axis=1)\n    train = train.drop(columns_to_drop_train, axis=1)\n    \n    feature_info['columns_to_drop_train'] = columns_to_drop_train\n    feature_info['columns_to_drop_test'] = columns_to_drop_test\n    \n    train = transform_replace_value(train, mapping)\n    test = transform_replace_value(test, mapping)\n    \n    train = extract_presence_flags(train)\n    test = extract_presence_flags(test)\n    \n    \n        \n    y_train = transform_add_is_present_flag(train['totals.transactionRevenue'], 0, res_type=np.int64)\n#     y_train = list(map(lambda p:p.values, y_train))\n    train = train.drop('totals.transactionRevenue', axis=1)\n    \n    \n    train = add_features(train, params)\n    test = add_features(test, params)\n    \n    train, train_ids = drop_ids(train)\n    test, test_ids = drop_ids(test) \n    \n\n    \n#     train, test, feature_values = utl.drop_rare_categories(train, test, params['min_freq'], params['min_rate'])\n#     feature_info['cat_feature_values'] = feature_values\n    \n    \n    train, test, indexers = factorize(train, test, params)\n    \n    feature_info['indexers'] = indexers\n    feature_info['feature_names'] = copy.deepcopy(train.columns)\n    \n#     train = train.values\n#     test = test.values\n    \n    \n    return train, y_train, test, train_ids, test_ids, feature_info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c606b35e2d4bf8578a5a00cd1d3150bb967fe5b6"},"cell_type":"code","source":"from sklearn.model_selection import GroupKFold\n\ndef get_folds(df=None, n_splits=5):\n    \"\"\"Returns dataframe indices corresponding to Visitors Group KFold\"\"\"\n    # Get sorted unique visitors\n    unique_vis = np.array(sorted(df['fullVisitorId'].unique()))\n\n    # Get folds\n    folds = GroupKFold(n_splits=n_splits)\n    fold_ids = []\n    ids = np.arange(df.shape[0])\n    for trn_vis, val_vis in folds.split(X=unique_vis, y=unique_vis, groups=unique_vis):\n        fold_ids.append(\n            [\n                ids[df['fullVisitorId'].isin(unique_vis[trn_vis])],\n                ids[df['fullVisitorId'].isin(unique_vis[val_vis])]\n            ]\n        )\n\n    return fold_ids\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97e43de3523722bf527444b415094b33d9461868"},"cell_type":"code","source":"train, y_train, test, train_ids, test_ids, feature_info = transform_dataset(train, test,  params={})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1234750f80b8bfe511ccfdfe9cea8235096dcb0"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dec05686f1f1376019ddcbb42eb83cbdc0a2216d"},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84c8ce25f9e6042455012776460b4605dccbc6d4"},"cell_type":"code","source":"len(feature_info['feature_names'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8beebe8344728f724ec1b00139c0561a94b1f7d"},"cell_type":"code","source":"feature_info['feature_names']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63e26526efb112b637d00feb1eb0f056d69a027c"},"cell_type":"code","source":"# train['weekday']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"395044a74857a2e3da5afe427c997b0831173b21"},"cell_type":"code","source":"y_train[0].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6071a0b3ab1f022241a821949c4a66435726a1ab"},"cell_type":"code","source":"y_train[1].shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff6131827ea0a4efdfe9bdce126af23ed4f1f30c"},"cell_type":"code","source":"folds = get_folds(df=train_ids, n_splits=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31a71321e2dc73838278a6b336b0f5a0abde34d8"},"cell_type":"code","source":"\n\nparam = {'num_leaves': 300,\n         'min_data_in_leaf': 30, \n         'objective':'regression',\n         'max_depth': -1,\n         'learning_rate': 0.005,\n         \"min_child_samples\": 20,\n         \"boosting\": \"gbdt\",\n         \"feature_fraction\": 0.9,\n         \"bagging_freq\": 1,\n         \"bagging_fraction\": 0.8 ,\n         \"bagging_seed\": 11,\n         \"metric\": 'rmse',\n         \"lambda_l1\": 1,\n         \"verbosity\": 1}\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90e4e398841ff4ac65cc507e2de65a895dabf060"},"cell_type":"code","source":"train_df = train\ntest_df = test\noof = np.zeros(len(train_df))\npredictions = np.zeros(len(test_df))\nstart = time.time()\nfeatures = list(train_df.columns)\nfeature_importance_df = pd.DataFrame()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3f73596c51cb08d5558bdbf96fc5eb1a38e7cbb"},"cell_type":"code","source":"categorical_feats = list(feature_info['indexers'].keys())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cace1539589c5bc010d8aaeea8c3a022349a8fe5"},"cell_type":"code","source":"train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce2f4d7fd3fb2110bb70514fcea026494e32287f"},"cell_type":"code","source":"lgb = lgbm\nregressors = []\n\nfor fold_, (trn_idx, val_idx) in enumerate(folds):\n    trn_data = lgb.Dataset(train_df.iloc[trn_idx], label=np.log(y_train[0].iloc[trn_idx] + 1), categorical_feature=categorical_feats)\n    val_data = lgb.Dataset(train_df.iloc[val_idx], label=np.log(y_train[0].iloc[val_idx] + 1), categorical_feature=categorical_feats)\n    \n    num_round = 10000\n    clf = lgb.train(param, trn_data, num_round, valid_sets = [trn_data, val_data], verbose_eval=100, early_stopping_rounds = 100)\n    regressors.append(clf)\n    oof[val_idx] = clf.predict(train_df.iloc[val_idx], num_iteration=clf.best_iteration)\n    \n    fold_importance_df = pd.DataFrame()\n    fold_importance_df[\"feature\"] = features\n    fold_importance_df[\"importance\"] = clf.feature_importance()\n    fold_importance_df[\"fold\"] = fold_ + 1\n    feature_importance_df = pd.concat([feature_importance_df, fold_importance_df], axis=0)\n    \n    predictions += clf.predict(test_df, num_iteration=clf.best_iteration) / len(folds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"659157d7e535b3127975ad6fbca30b45849825ca"},"cell_type":"code","source":"fig, ax = plt.subplots(figsize=(12,18))\nlgb.plot_importance(clf, max_num_features=50, height=0.8, ax=ax)\nlgb_features = clf.feature_importance\nax.grid(False)\nplt.title(\"LightGBM - Feature Importance\", fontsize=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5249429291ad9479551b140a612c27661a2d3afb"},"cell_type":"code","source":"test_pred = make_prediction(test_ids['fullVisitorId'], np.maximum(np.exp(predictions) - 1, 0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e451ee5c3930dc308fdd61088ea5579aaf8c7b5"},"cell_type":"code","source":"res_p = make_result_df(test_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eaf9c1a5724bfab6ac8c2c86010d891a76bea5da"},"cell_type":"code","source":"write_df(res_p, './prediction.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0631ce3c0d0c04ce346582136cea591574f184a4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"41cf8a1f8e6975f3283336876a15113989440b28"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}