{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom collections import defaultdict\nfrom collections import OrderedDict\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\n\nfrom tqdm import tqdm\nimport lightgbm as lgb\nimport matplotlib.pyplot as plt\nimport json\nimport pickle\nimport sys\nimport math\nimport gc\n\nfrom pandas.io.json import json_normalize\nfrom datetime import datetime\n\nimport os\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"294c61bbe265309fe0b0a62d68124878346af928"},"cell_type":"markdown","source":"## 1. Data loading"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def load(csv_path, nrows=None):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    df = pd.read_csv(csv_path, \n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str'}, # Important!!\n                     nrows=nrows,)\n    \n    for column in JSON_COLUMNS:\n        column_as_df = json_normalize(df[column])\n        column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n        df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61bd2d70249cf8ac9c37182d644148a232eb16b8"},"cell_type":"code","source":"train = load('../input/train.csv')\ntest = load('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60c5c4baba11cd4169051a7ea38e184d1f958c56"},"cell_type":"code","source":"def to_numeric(data):\n    for feature in data.dtypes[data.dtypes == 'object'].index: \n        data[feature] = pd.to_numeric(data[feature],errors='ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e904695c74194796d4c261688f749b2a66e20f"},"cell_type":"code","source":"to_numeric(train)\nto_numeric(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2dc0d36fb29cb516b0cf575a91904198a1e73264"},"cell_type":"code","source":"# Customers present in train and test\nrepeat_clients = set(train.fullVisitorId) & set(test.fullVisitorId)\nlen(set(train.fullVisitorId)), len(repeat_clients)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf0261bb69aa352d1a5592b9ad455b87b901727e"},"cell_type":"code","source":"train['isBuy'] = train['totals.transactionRevenue'] > 0 \ntrain['isBuy'].mean(), train[train.fullVisitorId.isin(repeat_clients)].isBuy.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e794a12dac4bc070e640acbf344de7d5de72a696"},"cell_type":"code","source":"# Drop columns with constant value \nconst_columns = [column for column in train.columns if len(train[column].value_counts(dropna=False)) == 1]\ntrain.drop(columns=const_columns, axis=1, inplace=True)\ntest.drop(columns=const_columns, axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0d3ddb3ef7e925abbe8b6da42efb98c03821950"},"cell_type":"code","source":"# Drop columns containing only in train\nset(train.columns) ^ set(test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc9a7ea3986775c3bbca542ea27e7689c02d6783"},"cell_type":"code","source":"del train['trafficSource.campaignCode']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"408d337574f1b1e2a334a19d1870c532d6f9ae6b"},"cell_type":"markdown","source":"## 2. Feature generation"},{"metadata":{"trusted":true,"_uuid":"87b66dc257003b4fc94fc2f426515c3b7089eb7d"},"cell_type":"code","source":"def get_time_feature(df):\n    df.date = pd.to_datetime(df.date, format='%Y%m%d')\n    df.visitStartTime = pd.to_datetime(df.visitStartTime, unit='s')\n    df['hour'] = df.visitStartTime.dt.hour\n    df['dayofweek'] = df.visitStartTime.dt.dayofweek\n    df['weekofyear'] = df.visitStartTime.dt.weekofyear\n    df['month'] = df.visitStartTime.dt.month","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4caff3ce8890a45e4478813315649aeda4c7f09"},"cell_type":"code","source":"get_time_feature(train)\nget_time_feature(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cf472cf719c3893dddf80bbdad97e2cf6de369a"},"cell_type":"code","source":"class ThrColumnEncoder:\n    \"\"\"\n    The threshold label encoder. \n    To avoid overfitting we can combine rare values into one group. Class work with pd.Series.\n    thr: Threshold as a percentage, values whose number is less than the threshold will be replaced by a single label.\n    \"\"\"\n    def __init__(self, thr=0.5):\n        self.thr = thr\n        self.categories = defaultdict(lambda:-1) # Those values that are X_test, but not in X_train will be replaced by -1.\n        \n    def fit(self, x):\n        values = x.value_counts(dropna=False)*100/len(x)\n        for value, key in enumerate(values[values >= self.thr].index):\n            self.categories[key] = value\n        for value, key in enumerate(values[values < self.thr].index):\n            self.categories[key] = -1 # Rare values replace -1\n            \n    def transform(self, x):   \n        return x.apply(self.categories.get)\n    \n    def fit_transform(self, x):\n        self.fit(x)\n        return self.transform(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ad409e9111434344689a2e14a4de3de67ba8890"},"cell_type":"code","source":"class ThrLabelEncoder:\n    \"\"\"\n    Work with pd.DataFrame.\n    \"\"\"\n    def __init__(self, thr=0.5):\n        self.thr = thr\n        self.column_encoders = {}\n        self.features = None\n        \n    def fit(self, X):\n        self.features = X.columns\n        for feature in self.features:\n            ce = ThrColumnEncoder()\n            ce.fit(X[feature])\n            self.column_encoders[feature] = ce\n            \n    def transform(self, X):\n        X = X.copy()\n        for feature in self.features: \n            ce = self.column_encoders[feature]\n            X.loc[:, feature] = ce.transform(X[feature])\n        return X\n            \n    def fit_transform(self, X):\n        self.fit(X)\n        return self.transform(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"53ced9ab8b17c936c726f1992f0d1e89985abee6"},"cell_type":"code","source":"id_cols = 'fullVisitorId', 'sessionId'\n\ntfidf_cols = 'geoNetwork.networkDomain', 'trafficSource.referralPath', 'trafficSource.source'\n\ncat_cols = list(set(train.dtypes[train.dtypes == 'object'].index) - set(id_cols) - set(tfidf_cols)  |  set(('dayofweek', 'hour', 'month')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6c2a7d4ede3a259ace3f375c77fa468fe6ab84e"},"cell_type":"code","source":"class DummyEncoder(ThrLabelEncoder):\n    \"\"\"\n    For each unique value of a categorical feature, we can create our own binary feature.\n    Generally speaking, this transformation is not necessary for gradient boosting.\n    But for this task, I decided to move from the feature description of the visit to the feature description\n    of the client and without the dummy-encoding can not do it.\n    \"\"\"\n    def transform(self, X):\n        result = []\n        for feature in self.features: \n            ce = self.column_encoders[feature]\n            tf_feature = ce.transform(X[feature])\n            popular_values = [value for key, value in self.column_encoders[feature].categories.items() if value != -1]\n            popular_keys = [key for key, value in self.column_encoders[feature].categories.items() if value != -1]\n            columns = ['%s_%s'%(feature, key) for key in popular_keys + ['rare']]\n            feature_dummies = pd.concat([tf_feature == value for value in popular_values] + [tf_feature == -1], axis=1)\n            feature_dummies.columns = columns\n            result.append(feature_dummies)\n        return pd.concat(result, axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4503e9dc146d1109f0dc924863c28c496964c51"},"cell_type":"code","source":"de = DummyEncoder()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dcc5a96f9b803b3e8f91b87f005be02fdfc05002"},"cell_type":"code","source":"train_de = de.fit_transform(train[cat_cols])\ntest_de = de.transform(test[cat_cols])\n\ntrain_de.index = train.fullVisitorId\ntest_de.index = test.fullVisitorId","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"01413c3729faccb5699c2b250bc3b6b8c8fbfa7a"},"cell_type":"markdown","source":"##  3. Aggregation of data to the client level"},{"metadata":{"trusted":true,"_uuid":"a0450b3d0df2128aebdffaa6871628a1fc8f74fa"},"cell_type":"code","source":"def life_time(x):\n    return (x.max() - x.min()).total_seconds()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57555de6166ea92b021d3e881353603eb105ba67"},"cell_type":"code","source":"aggregates = {'totals.pageviews': [sum, min, max, np.mean], \n              'totals.hits': [sum, min, max, np.mean], \n              'date': life_time,\n              'visitNumber': [max, min],\n              'totals.bounces': sum,\n              'totals.transactionRevenue': sum\n             }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b912fe7d5d785b5cf61209489f1d205dd93cffa8"},"cell_type":"code","source":"%time train_gr = train.groupby('fullVisitorId').agg(aggregates)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"482b3c77242e0d09be93fa94b07725c152f9b490"},"cell_type":"code","source":"def groupby_rename(df):\n    df.columns = ['%s_%s'% (df.columns.levels[0][i],df.columns.levels[1][j]) for i,j in \\\n                  zip(df.columns.labels[0], df.columns.labels[1])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09c8c983936d3ec49ff6f7ff74862c97ef390244"},"cell_type":"code","source":"groupby_rename(train_gr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"df19a27019db73f4c5dec96bece6a327f6274a82"},"cell_type":"code","source":"del aggregates['totals.transactionRevenue']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"149fa708481e45087130bf5bb66818944b970db3"},"cell_type":"code","source":"%time test_gr = test.groupby('fullVisitorId').agg(aggregates)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0dbe94b30188ef0b7badbf3f1f34bcdd5c6cf72"},"cell_type":"code","source":"groupby_rename(test_gr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98b9e3d20de38416c47e541a036bf65d61b77920"},"cell_type":"code","source":"train_de_sum = train_de.groupby(level=0).sum()\n\ntest_de_sum = test_de.groupby(level=0).sum()\n\nX = pd.concat([train_de_sum, train_gr], axis=1)\nX_test = pd.concat([test_de_sum, test_gr], axis=1)\n\nY = np.log1p(X['totals.transactionRevenue_sum'])\ndel X['totals.transactionRevenue_sum']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b96108b2849b4d520188ddbf24c1df6e82455864"},"cell_type":"code","source":"del train_de_sum\ndel test_de_sum\ndel train_gr\ndel test_gr\ndel train_de\ndel test_de","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d02acb7712467be791c494104b60e07c569743e4"},"cell_type":"code","source":"class TFIDFER:\n    \"\"\"\n    Class encapsulating tfidf transformation and renaming column of pd.Dataframe with using name tfidf-features.\n    \"\"\"\n    def __init__(self, max_df=0.9, min_df=0.01, max_features=100, ngram_range=(1,2)):\n        self.max_df = max_df\n        self.min_df = min_df\n        self.max_features  = max_features\n        self.ngram_range = ngram_range\n        \n        self.column_encoders = {}\n        self.features = None\n        \n    def fit(self, X): \n        self.features = X.columns\n        for feature in self.features:\n            tfidf = TfidfVectorizer(max_df=self.max_df, min_df=self.min_df, max_features=self.max_features, ngram_range=self.ngram_range)\n            tfidf.fit(X[feature])\n            self.column_encoders[feature] = tfidf\n            \n    def transform(self, X):\n        result = []\n        for feature in self.features: \n            items_tfidf = pd.DataFrame(self.column_encoders[feature].transform(X[feature]).toarray(), X.index)\n            col_names = [word.replace(' ','_') for word, index in sorted(self.column_encoders[feature].vocabulary_.items(), key = lambda x:x[1])]\n            items_tfidf.columns =  ['tfidf_%s_%s'%(feature, name) for name in col_names] \n            result.append(items_tfidf.copy())\n        return pd.concat(result, axis=1)\n            \n    def fit_transform(self, X):\n        self.fit(X)\n        return self.transform(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e37dc08300595e5907692d441f15adc6b7d6692d"},"cell_type":"code","source":"def get_tfidf(df):\n    df['geoNetwork.networkDomain'] = df['geoNetwork.networkDomain'].apply(lambda x: x.replace('.', ' ').replace(':', ' ') + ' ')\n    df['trafficSource.source'] = df['trafficSource.source'].apply(lambda x: x.replace('.', ' ').replace(':', ' ') + ' ')\n    df['trafficSource.referralPath'] = df['trafficSource.referralPath'].astype(str).apply(lambda x: x.replace('/', ' ') + ' ')\n    aggregates = {'geoNetwork.networkDomain': sum, \n                  'trafficSource.source': sum, \n                  'trafficSource.referralPath': sum}\n    return df.groupby('fullVisitorId').agg(aggregates)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"804f22688be70116cb32557b2dd3a9da485dca88"},"cell_type":"code","source":"aggregates = {'geoNetwork.networkDomain': sum, \n              'trafficSource.source': sum, \n              'trafficSource.referralPath': sum}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2125bea19d34265ea4c8660d95deb0e451978e4b"},"cell_type":"code","source":"%time train_ftidf_sum = get_tfidf(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2c6fe3baf60894145f77f08566a2eb076f9e1bd"},"cell_type":"code","source":"%time test_ftidf_sum = get_tfidf(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a0d243420d0f46690ea5f299e2e527f81f02bae"},"cell_type":"code","source":"tfidfer = TFIDFER()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"101e23864638e03264ec06b79e59f34c14063251"},"cell_type":"code","source":"train_tfidf = tfidfer.fit_transform(train_ftidf_sum)\ntest_tfidf = tfidfer.transform(test_ftidf_sum)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10d49b5cbc2a829f2a39886b8ceebb76e94988fb"},"cell_type":"code","source":"X_test = pd.concat([X_test, test_tfidf], axis=1)\nX = pd.concat([X, train_tfidf],axis =1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d177fe4531a471338617ce3804cd7ed29bf71c01"},"cell_type":"code","source":"del train \ndel test \ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bc45edfbc3e66ac6065b0a972b7d97432a27724"},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f4a57c203cb76b386d02632f1b46dae57eba17b"},"cell_type":"markdown","source":"## 4. Model building"},{"metadata":{"trusted":true,"_uuid":"de30b90b97a16f73085968d6bc25acfb29dddc08"},"cell_type":"code","source":"X_train, X_valid, Y_train, Y_valid = train_test_split(X, Y, test_size=0.33, random_state=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8ecbe6bdf7fdd63ba5c42ff3a7ed9421e23eee4"},"cell_type":"code","source":"gbm = lgb.LGBMRegressor(objective = 'regression',  \n                        max_depth = 11,\n                        colsample_bytre = 0.8,\n                        subsample = 0.8, \n                        learning_rate = 0.1,\n                        n_estimators = 300)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4297cf7668338de1f582aca9018001bb38515fab","collapsed":true},"cell_type":"code","source":"gbm.fit(X_train, Y_train, \n        eval_set=[(X_valid, Y_valid)],\n        eval_metric='rmse',\n        early_stopping_rounds=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca37dd06a8dce462a2e5d659e2e81e58b7110d28"},"cell_type":"code","source":"%matplotlib inline\nlgb.plot_importance(gbm, max_num_features=120, figsize=(10,40))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d68f9ac39cb2c0bfb2b7cd787311d3be96acc6d"},"cell_type":"code","source":"Y_test = pd.Series(gbm.predict(X_test),index= X_test.index)\n\nY_test[Y_test<0] = 0\n\nY_test.name = \"PredictedLogRevenue\"\n\nY_test.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}