{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":false,"_kg_hide-output":false},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport os\nimport gc\nimport re\n\nimport json\nfrom pandas.io.json import json_normalize\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom tqdm import trange,tqdm\nfrom collections import Counter\nfrom functools import reduce\n\npd.options.display.max_rows = 8\npd.options.display.max_columns = 999\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"383433bda3cc09260db48895f2c64e24264515f5"},"cell_type":"markdown","source":"# check ids"},{"metadata":{"trusted":true,"_uuid":"00671d5fbc5a4eebd294225942b3895193ebb4cf"},"cell_type":"code","source":"id_col = 'fullVisitorId'\ncols = [id_col, 'visitStartTime']\n\ntest_df = pd.read_csv('../input/test_v2.csv', dtype={id_col: 'str'}).loc[:, cols].copy()\nprint(test_df.shape)\nsub_df = pd.read_csv('../input/sample_submission_v2.csv', dtype={id_col: 'str'})\nprint(sub_df.shape)\ns = test_df[id_col].unique()\nsub_ids = sub_df[id_col].unique()\nprint(s.shape, sub_ids.shape)\nsub_ids = np.intersect1d(s, sub_ids)\ndel sub_df, s\ngc.collect()\nprint(sub_ids.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a33ac552f6cd8e79c1a3a4ff0d997162a83e95d"},"cell_type":"code","source":"def read_cols(file_tag, cols):\n    data_reader = pd.read_csv(f'../input/{file_tag}_v2.csv', iterator=True, chunksize=20000,\n                          dtype={id_col: 'str'})\n    df = pd.DataFrame()\n    for it in data_reader:\n        df = df.append(it[cols].copy(), ignore_index=True)\n        gc.collect()\n    del data_reader, it\n    gc.collect()\n    \n    return df\n\n\ntrain_df = read_cols('train', cols)\ngc.collect()\nprint(train_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7174b13f6bba69bbf48116e5da6fbe653512dbf9"},"cell_type":"code","source":"cids = np.intersect1d(train_df[id_col], test_df[id_col])\nprint(f'cids: {cids.shape}')\n\ndf = train_df.append(test_df, ignore_index=True)\ngc.collect()\nprint(df.shape)\ncnt = df[id_col].value_counts()\nprint(cnt.shape)\ntr_ids = cnt.loc[cnt>1].index.values\nprint(f'tr_ids: {tr_ids.shape}')\ntr_df = df.loc[df[id_col].isin(tr_ids)]\ngc.collect()\nprint(tr_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"61d067686ae9f37216b1b251e9f8c2062507e40a"},"cell_type":"code","source":"tm_col = 'visitStartTime'\nprint(df[tm_col].nunique())\n\ntime_frame_start1 = pd.to_datetime('2017-05-01').timestamp()\ntime_frame_end1 = pd.to_datetime('2017-10-01').timestamp()\ntime_frame_start2 = pd.to_datetime('2018-01-01').timestamp()\ntime_frame_end2 = pd.to_datetime('2018-06-01').timestamp()\n\ntr_ids1 = cnt.loc[1==cnt].index.values\nprint(tr_ids1.shape)\n\ntr_df1 = df.loc[df[id_col].isin(tr_ids1) & (df[tm_col]>=time_frame_start1) & (df[tm_col]<time_frame_end1)]\ntr_df2 = df.loc[df[id_col].isin(tr_ids1) & (df[tm_col]>=time_frame_start2) & (df[tm_col]<time_frame_end2)]\nprint(tr_df1.shape, tr_df2.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"56fd3ed907294cf7d1ac62bd6cda78e67e65618d"},"cell_type":"code","source":"tr_ids1 = tr_df1[id_col].unique()\ntr_ids2 = tr_df2[id_col].unique()\ntr_ids = reduce(np.union1d, (tr_ids, tr_ids1, tr_ids2))\nprint(f'tr_ids: {tr_ids.shape}')\ndf = df.loc[df[id_col].isin(tr_ids)]\ngc.collect()\nprint(df.shape)\ndel df,train_df,test_df,cnt,tr_ids1,tr_ids2,tr_df1,tr_df2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"# load & flatten train data"},{"metadata":{"trusted":true,"_uuid":"433c21c52cdfc1acc625c0096112271127a90afe"},"cell_type":"code","source":"def extract_hits(df):\n    def _extract_hits(rec):\n        target = rec['target']\n        info = re.sub(r\"\\\\?['\\\"]\", \"'\", rec['hits'])\n\n        meta_infos = re.findall(r\"'time'\\:\\s*'(\\d+)'.*?exitScreenName.*?/([^/]+?)',\", info)\n        times,exitScreenNames = zip(*meta_infos) if meta_infos else ([],[])\n        times = [int(t) for t in times] if times else []\n        exitScreenNames = ' '.join(set(exitScreenNames)) if exitScreenNames else 'None'\n\n        promoNames = set(re.findall(r\"promoName'\\:\\s*'(.+?)',\", info))\n        promoNames = ' '.join(promoNames) if promoNames else 'None'\n\n        products = re.findall(\"v2ProductName'\\:\\s*'(.+?)',.*?v2ProductCategory'\\:\\s*'(.+?)',.*?'productPrice'\\:\\s*'(\\d+)',\", info)\n        productNames,productCategorys,productPrices = zip(*products) if products else ([],[],[])\n        productPrices = [int(price) for price in productPrices] if productPrices else []\n        products = dict(zip(productNames, productPrices))\n        productNames = ' '.join(products.keys()) if products else 'None'\n        productPrices = list(products.values())\n        productCategorys = ' '.join(set(productCategorys)) if products else 'None'\n        hitProductNames = [name for name,price in products.items() if price==target] if target>0 and products else []\n        hitProductNames = ' '.join(hitProductNames) if hitProductNames else 'None'\n\n        hit_cnt = len(times)\n        total_time = np.max(times) if times else 0\n        mean_time = total_time//(hit_cnt-1) if hit_cnt>1 else 0\n        diff_times = np.diff(times)\n        mean_diff_time,min_diff_time,max_diff_time,std_diff_time = 0,0,0,0\n        if diff_times.shape[0]>0:\n            mean_diff_time = int(np.mean(diff_times))\n            min_diff_time = np.min(diff_times)\n            max_diff_time = np.max(diff_times)\n            std_diff_time = int(np.std(diff_times))\n\n        mean_price = int(np.mean(productPrices)) if productPrices else 0\n        min_price = np.min(productPrices) if productPrices else 0\n        max_price = np.max(productPrices) if productPrices else 0\n        std_price = int(np.std(productPrices)) if productPrices else 0\n        mean_price_ratio = target/mean_price if mean_price else -1.0\n        min_price_ratio = target/min_price if min_price else -1.0\n        max_price_ratio = target/max_price if max_price else -1.0\n\n        return (total_time,mean_time,mean_diff_time,min_diff_time,max_diff_time,std_diff_time,exitScreenNames,promoNames,\n                productNames,productCategorys,mean_price,min_price,max_price,std_price,mean_price_ratio,min_price_ratio,\n                max_price_ratio,hitProductNames)    \n    \n    (df['total_time'],df['mean_time'],df['mean_diff_time'],df['min_diff_time'],df['max_diff_time'],df['std_diff_time'],\n     df['exitScreenNames_src'],df['promoNames_src'],df['productNames_src'],df['productCategorys_src'],df['mean_price'],\n     df['min_price'],df['max_price'],df['std_price'],df['mean_price_ratio'],df['min_price_ratio'],df['max_price_ratio'],\n     df['hitProductNames_src']) = zip(*df[['hits','target']].apply(_extract_hits, axis=1))\n    \n    return df.drop('hits', axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e50a07e3de4cb9a97ff37f7da93a5c3765e70103"},"cell_type":"code","source":"def load_df(ids, data_tag='train'):\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    \n    data_reader = pd.read_csv(f'../input/{data_tag}_v2.csv', iterator=True, chunksize=100000,\n                              converters={column: json.loads for column in JSON_COLUMNS},\n                              dtype={id_col: 'str'})\n    df = pd.DataFrame()\n    for data in data_reader:\n        data = data.loc[data[id_col].isin(ids)].reset_index(drop=True).copy()\n        if data.shape[0] > 0:\n            print(f'before: {data.shape}', end='\\t')\n            for column in JSON_COLUMNS:\n                column_as_df = json_normalize(data[column])\n                column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n                data = data.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n                del column_as_df\n                gc.collect()\n            \n            data = data.rename(columns={'totals.transactionRevenue':'target'})\n            data['target'] = data['target'].fillna(0) if 'target' in data.columns else 0\n            data.target = data.target.astype(np.int64)\n            gc.collect()\n            data = extract_hits(data)\n            gc.collect()\n            \n            df = df.append(data, ignore_index=True, sort=False)\n            print(f'after: {data.shape}, total: {df.shape}')\n            del data\n            gc.collect()\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7028371b4368fc25b72facc3ad63b5bb4a50291d"},"cell_type":"code","source":"train_df_r = load_df(tr_ids)\nprint(f'train_df_r: {train_df_r.shape}')\ngc.collect()\ntest_df = load_df(tr_ids, 'test')\nprint(f'test_df: {test_df.shape}')\ngc.collect()\ntrain_df_r = train_df_r.append(test_df, ignore_index=True, sort=False)\ndel test_df\ngc.collect()\nprint(f'train_df_r: {train_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0b3513a772ea40388fa54b5ee575da9bd9532a3"},"cell_type":"code","source":"def find_still_cols(df):\n    still_cols = []\n    for col in df.columns:\n        cnt = df[col].nunique(dropna=False)\n        if cnt <= 1:\n            still_cols.append(col)\n    return still_cols","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e9ef81daffc95ae3833c2e5c6e30261a4da32a7"},"cell_type":"code","source":"still_cols = ['trafficSource.adwordsClickInfo.gclId', 'trafficSource.campaignCode'] + find_still_cols(train_df_r)\nprint(f'still_cols(len(still_cols)): {still_cols}')\ntrain_df_r = train_df_r.drop(still_cols, axis=1)\ngc.collect()\nprint(f'train_df_r: {train_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c3b8eb3c7fded799f1f6352e60f0374dc195715d"},"cell_type":"markdown","source":"# record level"},{"metadata":{"_uuid":"393b0c04e2cc8412d7d798655b3cb648081fe91a"},"cell_type":"markdown","source":"## stat pv"},{"metadata":{"trusted":true,"_uuid":"ea927087f26ff35f17473286ce9ed532379f67ea"},"cell_type":"code","source":"def encode_user_pv(_df, gp_col=id_col, _col_prefix='user', sort_col='visitStartTime', \n                   session_interval=7*24*3600, window_interval=60*24*3600):\n    # global scope\n    _df = _df.sort_values(by=[gp_col, sort_col])\n    gp = _df.groupby(gp_col)[sort_col]\n    _df[f'{_col_prefix}_g_span'] = gp.diff().fillna(0)\n    _df = _df.join(gp.count(), on=gp_col, rsuffix='_g_pv').rename(\n        columns={f'{sort_col}_g_pv': f'{_col_prefix}_g_pv'})\n    \n    # window scope\n    _df = _df.join(gp.max(), on=gp_col, rsuffix='_rm').rename(columns={f'{sort_col}_rm': 'window_id'})\n    _df['window_id'] = ((_df['window_id'] - _df[sort_col]) // window_interval).astype(np.uint8)\n    gp_cols = [gp_col, 'window_id']\n    gp = _df.groupby(gp_cols)\n    _df = _df.join(gp[sort_col].count(), on=gp_cols, rsuffix='_w_pv').rename(\n        columns={f'{sort_col}_w_pv': f'{_col_prefix}_w_pv'})\n    _df = _df.join(gp[sort_col].max()-gp[sort_col].min(), on=gp_cols, rsuffix='_w_span').rename(\n        columns={f'{sort_col}_w_span': f'{_col_prefix}_w_span'})\n    _df = _df.join(gp[f'{_col_prefix}_g_span'].first(), on=gp_cols, rsuffix='_w_idle').rename(\n        columns={f'{_col_prefix}_g_span_w_idle': f'{_col_prefix}_w_idle'})\n    \n    # session scope\n    _df['session_id'] = _df[f'{_col_prefix}_g_span']\n    _df.loc[_df['session_id']<=session_interval, 'session_id'] = 0\n    _df.loc[_df['session_id']>session_interval, 'session_id'] = 1\n    _df['session_id'] = _df.groupby(gp_col)['session_id'].cumsum().astype(np.uint16)\n    gp_cols = [gp_col, 'session_id']\n    gp = _df.groupby(gp_cols)\n    _df = _df.join(gp[sort_col].count(), on=gp_cols, rsuffix='_s_pv').rename(\n        columns={f'{sort_col}_s_pv': f'{_col_prefix}_s_pv'}) \n    _df = _df.join(gp[sort_col].max()-gp[sort_col].min(), on=gp_cols, rsuffix='_s_span').rename(\n        columns={f'{sort_col}_s_span': f'{_col_prefix}_s_span'})\n    _df = _df.join(gp[f'{_col_prefix}_g_span'].first(), on=gp_cols, rsuffix='_s_idle').rename(\n        columns={f'{_col_prefix}_g_span_s_idle': f'{_col_prefix}_s_idle'})\n    \n    # visit scope\n    gp_cols = [gp_col, 'visitId']\n    gp = _df.groupby(gp_cols)\n    _df = _df.join(gp[sort_col].count(), on=gp_cols, rsuffix='_v_pv').rename(\n        columns={f'{sort_col}_v_pv': f'{_col_prefix}_v_pv'}) \n    _df = _df.join(gp[sort_col].max()-gp[sort_col].min(), on=gp_cols, rsuffix='_v_span').rename(\n        columns={f'{sort_col}_v_span': f'{_col_prefix}_v_span'})\n    _df = _df.join(gp[f'{_col_prefix}_g_span'].first(), on=gp_cols, rsuffix='_v_idle').rename(\n        columns={f'{_col_prefix}_g_span_v_idle': f'{_col_prefix}_v_idle'})\n    \n    for flag in list('gwsv'):\n        _df[f'{_col_prefix}_{flag}_pv'] = _df[f'{_col_prefix}_{flag}_pv'].astype(np.int16)\n        _df[f'{_col_prefix}_{flag}_span'] = _df[f'{_col_prefix}_{flag}_span'].astype(np.int32)\n    for flag in list('wsv'):\n        _df[f'{_col_prefix}_{flag}_idle'] = _df[f'{_col_prefix}_{flag}_idle'].astype(np.int32)\n    _df = _df.sort_index()\n\n    return _df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e637342b31ea995e739bd2b2c5690fc2c5f5da9"},"cell_type":"code","source":"train_df_r = encode_user_pv(train_df_r)\ngc.collect()\nprint(f'train_df_r: {train_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"35390427da6b5148408322b57e80f0da13c2fefe"},"cell_type":"markdown","source":"## encode obj"},{"metadata":{"trusted":true,"_uuid":"e4e7e078cef26c76b05a94d7bb38074c1e970885"},"cell_type":"code","source":"print(list(train_df_r['device.browser'].unique()))\nprint(list(train_df_r['trafficSource.adContent'].unique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46e2fb0b5a46137411946bce01b4e741922fc50c"},"cell_type":"code","source":"def flatten_counter(cnts):\n    pairs = sorted(cnts.items(), key=lambda pair: (pair[1], pair[0]))\n    last_cnt = 0\n    new_pairs = []\n    for k, v in pairs:\n        if v > last_cnt:\n            new_pairs.append((k, v))\n            last_cnt = v\n        else:\n            last_cnt += 1\n            new_pairs.append((k, last_cnt))\n    return dict(new_pairs)\n\n\nclass CountEncoder:\n    def __init__(self, nan_value=-127, count_unique=True):\n        self.counter = None\n        self.nan_value = nan_value\n        self.unseen_code_value = nan_value\n        self.count_unique = count_unique\n\n    def fit(self, x, need_fill_na=True):\n        x = x.fillna(self.nan_value) if need_fill_na else x\n        self.counter = Counter(x)\n        unseen_code = None\n        if not x.loc[x == self.nan_value].shape[0]:\n            unseen_code = sorted(self.counter.items(), key=lambda pair: (pair[1], pair[0]), reverse=True)[0][0]\n        self.counter = flatten_counter(self.counter) if self.count_unique else self.counter\n        if unseen_code is not None:\n            self.unseen_code_value = self.counter[unseen_code]\n\n    def transform(self, x, need_fill_na=True):\n        x = x.fillna(self.nan_value) if need_fill_na else x\n        return x.apply(lambda ele: self.unseen_code_value if (\n            ele == self.nan_value or ele not in self.counter) else self.counter[ele])\n\n    def fit_transform(self, x):\n        x = x.fillna(self.nan_value)\n        self.fit(x, need_fill_na=False)\n        return self.transform(x, need_fill_na=False)\n    \n\ndef get_int_type(col):\n    max_val = col.max()\n    if max_val < 2 ** 7:\n        return np.int8\n    elif max_val < 2 ** 15:\n        return np.int16\n    elif max_val < 2 ** 31:\n        return np.int32\n    else:\n        return np.int64\n\n\ndef encode_obj(df, cols=None, encoders=None, data_types=None, count_unique=True):\n    if not encoders:\n        encoders = {}\n        data_types = {}\n        for col in np.intersect1d(df.columns, cols):\n            cer = CountEncoder(count_unique=count_unique)\n            df[col] = cer.fit_transform(df[col])\n            encoders[col] = cer\n            data_type = get_int_type(df[col])\n            data_types[col] = data_type\n            df[col] = df[col].astype(data_type)\n    else:\n        for col in np.intersect1d(df.columns, cols):\n            df[col] = encoders[col].transform(df[col]).astype(data_types[col])\n\n    return df, encoders, data_types","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6823f7d121891915487da31b649155258fee867c"},"cell_type":"code","source":"obj_cols = ['channelGrouping', 'customDimensions', 'device.browser', 'device.deviceCategory', 'device.isMobile', \n            'device.operatingSystem', 'geoNetwork.city', 'geoNetwork.continent', 'geoNetwork.country', 'geoNetwork.metro',\n            'geoNetwork.networkDomain', 'geoNetwork.region', 'geoNetwork.subContinent', 'totals.bounces',\n            'totals.newVisits', 'trafficSource.adContent', 'trafficSource.adwordsClickInfo.adNetworkType',\n            'trafficSource.adwordsClickInfo.isVideoAd', 'trafficSource.adwordsClickInfo.slot', 'trafficSource.campaign',\n            'trafficSource.isTrueDirect', 'trafficSource.keyword', 'trafficSource.medium', 'trafficSource.referralPath',\n            'trafficSource.source']\n\ncol = 'trafficSource.keyword'\ntrain_df_r[col] = train_df_r[col].str.lower()\nfor col in ['device.browser','trafficSource.adContent','trafficSource.keyword','trafficSource.referralPath','trafficSource.source']:\n    train_df_r[f'{col}_src'] = train_df_r[col].fillna('unknown')\n\ntrain_df_r, cat_encoders, cat_data_types = encode_obj(train_df_r, obj_cols)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e58aca8a98bfd11ca5a1e4d4e35de4aefb686dec"},"cell_type":"code","source":"def convert_data_type(df):\n    int8_cols = ['totals.sessionQualityDim','totals.transactions','trafficSource.adwordsClickInfo.page']\n    for col in int8_cols:\n        df[col] = df[col].fillna(-127).astype(np.int8)\n\n    col = 'totals.pageviews'\n    df[col] = df[col].fillna(1)\n    int16_cols = ['visitNumber','totals.pageviews','totals.hits']              \n    for col in int16_cols:\n        df[col] = df[col].fillna(-127).astype(np.int16)\n\n    df.loc[df['totals.timeOnSite'].isnull() & (df['total_time']>0), 'totals.timeOnSite'] = np.round(\n        df.loc[df['totals.timeOnSite'].isnull() & (df['total_time']>0), 'total_time']/1000)\n    df = df.drop('total_time', axis=1)\n    gc.collect()\n    int32_cols = ['visitId','visitStartTime','totals.timeOnSite','mean_time','mean_diff_time','min_diff_time',\n                  'max_diff_time','std_diff_time']\n    for col in int32_cols:\n        df[col] = df[col].fillna(-127).astype(np.int32)\n\n    float32_cols = ['mean_price_ratio','min_price_ratio','max_price_ratio']\n    for col in float32_cols:\n        df[col] = df[col].fillna(-127).astype(np.float32)\n\n    col = 'totals.totalTransactionRevenue'\n    df[col] = df[col].fillna(0).astype(np.int64)\n    df['totals.extraRevenue'] = df[col] - df['target']\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12d1a95121c232e99350d807acf93776e106bc57"},"cell_type":"code","source":"train_df_r = convert_data_type(train_df_r)\ngc.collect()\nprint(f'train_df_r: {train_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"522ebc60c0287b2b616aecf0b3bf4184cbb00036"},"cell_type":"markdown","source":"### trafficSource.keyword_src"},{"metadata":{"trusted":true,"_uuid":"e80a50a7352aec0675ca334c539d3a13895c246d"},"cell_type":"code","source":"rec_num = 20\ncol = 'trafficSource.keyword_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0e2a2d56dac2b3ca7a80c70eed7cf98755fe844","scrolled":true},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode', min_df=2)\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09f2d2c7b63c6f68b74f91db12175f16f3e8240c"},"cell_type":"code","source":"tvr = TfidfVectorizer(strip_accents='unicode',analyzer='char',ngram_range=(3,5), max_df=0.8, min_df=5)\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3d36fc2b3b63db12af52e15768d0b0791147d10"},"cell_type":"markdown","source":"### trafficSource.referralPath_src"},{"metadata":{"trusted":true,"_uuid":"423169ade2383edd8138636c863beeb728276fac"},"cell_type":"code","source":"rec_num = 20\ncol = 'trafficSource.referralPath_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab0f809fba2380759b07fff0811927acadf52617","scrolled":true},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode', min_df=5)\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc43d4f3b6fd6b0871e239a88ee4601f580bbbe1"},"cell_type":"code","source":"col = 'trafficSource.referralPath_src'\ns = train_df_r[col].str.count('/')\nprint(s.loc[s<=0].shape)\nprint(s.quantile([0.7,0.8,0.9,0.95]))\n\n\ndef encode_ref_path(df, ref_encoders=None, ref_data_types=None):\n    col = 'trafficSource.referralPath_src'\n    \n    df['ref_level_cnt'] = df[col].str.count('/').astype(np.int8)\n    s = df[col].str.split('/')\n    df['ref_level_1'] = '/' + s.str.get(1)\n    for i in range(2, 5):\n        df[f'ref_level_{i}'] = df[f'ref_level_{i-1}'] + '/' + s.str.get(i)\n    df['ref_level_n'] = s.str.get(-1)\n    \n    cols=['ref_level_n'] + [f'ref_level_{i}' for i in range(1,5)]\n    if not ref_encoders:\n        df, ref_encoders, ref_data_types = encode_obj(df, cols)\n    else:\n        df, _, _ = encode_obj(df, cols, ref_encoders, ref_data_types)\n    \n    return df, ref_encoders, ref_data_types","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7af5e792624a49a194c0c0ab120158811a74d128"},"cell_type":"code","source":"train_df_r, ref_encoders, ref_data_types = encode_ref_path(train_df_r)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d755e6269568ea050c5e09806de059006eaf6806"},"cell_type":"markdown","source":"### trafficSource.source_src"},{"metadata":{"trusted":true,"_uuid":"b7803d82aa43b6eb7c211eadf7e443a0ca0fbf4b"},"cell_type":"code","source":"rec_num = 20\ncol = 'trafficSource.source_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15753285bf3b1bce9aca8010dbec480ac3935548"},"cell_type":"code","source":"s = train_df_r['trafficSource.source_src']\nss = np.sort(s.unique())\nprint(list(np.append(ss[:rec_num],ss[-rec_num:])))\n\ns = s.str.replace(r':\\d+\\s*$', '')\nss = np.sort(s.unique())\nprint(list(np.append(ss[:rec_num],ss[-rec_num:])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04bff349eb84abb20f926f99c06e893d008a7b69"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(s)\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a64b6d875db874b3151d8fdf2f50ef17a298f346"},"cell_type":"code","source":"col = 'trafficSource.source_src'\ns = train_df_r[col].str.count('\\.')\nprint(s.loc[s<=0].shape)\nprint(s.quantile([0.5,0.6,0.7,0.8,0.9,0.95,1]))\n\n\ndef encode_source(df, src_encoders=None, src_data_types=None):\n    col = 'trafficSource.source_src'\n    df[col] = df[col].str.replace(r':\\d+\\s*$', '')\n    \n    df['src_level_cnt'] = df[col].str.count('\\.').astype(np.int8) + 1\n    s = df[col].str.split('\\.')\n    for i in range(1, 4):\n        df[f'src_level_{i}'] = s.str.get(-i)\n    df['src_level_n'] = s.str.get(0)\n    \n    cols=['src_level_n'] + [f'src_level_{i}' for i in range(1,4)]\n    if not src_encoders:\n        df, src_encoders, src_data_types = encode_obj(df, cols)\n    else:\n        df, _, _ = encode_obj(df, cols, src_encoders, src_data_types)\n    \n    return df, src_encoders, src_data_types","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64a2ae5c23bb0351d73a02e2ca995712a9690971"},"cell_type":"code","source":"train_df_r, src_encoders, src_data_types = encode_source(train_df_r)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17d416682185186c35563f60f85a6a0d3c8a272b"},"cell_type":"markdown","source":"### device.browser_src"},{"metadata":{"trusted":true,"_uuid":"893379e14c01aafe0c0a7ce8930dcfa9baacc6f2"},"cell_type":"code","source":"rec_num = 20\ncol = 'device.browser_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d67ed1bfd53f993884c9efe764585f63c2e5482"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7b737dc54ce5e16968a976cc6f72f269d5d13e0"},"cell_type":"markdown","source":"### trafficSource.adContent_src"},{"metadata":{"trusted":true,"_uuid":"afb3233c6e2b5ca2b802c0c282635d28007c4490"},"cell_type":"code","source":"rec_num = 20\ncol = 'trafficSource.adContent_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8fb4b79f1a0ce81468a8f7b8edf7e2547eda077"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7af031abed5cf5e5ed4881cdf171562cc2fac1d4"},"cell_type":"markdown","source":"### exitScreenNames_src"},{"metadata":{"trusted":true,"_uuid":"59b108b62e7f288d63aef225aaf3832325e43568"},"cell_type":"code","source":"rec_num = 10\ncol = 'exitScreenNames_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5c730281865cc4e05092228516f0ffbd57dfff8"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a89ac7a7ff0d03f252b791fd7b8ced0e2770fffd"},"cell_type":"markdown","source":"### promoNames_src"},{"metadata":{"trusted":true,"_uuid":"3733153d066d497bc716d30ed14826edee808b77"},"cell_type":"code","source":"rec_num = 10\ncol = 'promoNames_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b74952a2b813dfec2dbdbce4571694fc30a679e1"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"615fd187ad0613a170948ffd2654b9329a4dce13"},"cell_type":"markdown","source":"### productNames_src"},{"metadata":{"trusted":true,"_uuid":"ee410970d09853377158d9b7b55e7787e5c3154a"},"cell_type":"code","source":"rec_num = 2\ncol = 'productNames_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fcc90f5c097c07f5424f3d29226455d26783aa8b"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2e4f731d04da7fe635afe14acb47156f5f9c5720"},"cell_type":"markdown","source":"### productCategorys_src"},{"metadata":{"trusted":true,"_uuid":"1a332ecf94adda1e49def0b5ceb135c113b52909"},"cell_type":"code","source":"rec_num = 10\ncol = 'productCategorys_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d6584039cf15233fa415b95649625e98d72ffa9"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f07f426185c5b33e82a579a01987a44958b7b522"},"cell_type":"markdown","source":"### hitProductNames_src"},{"metadata":{"trusted":true,"_uuid":"3b611f242efcfd2208152b78a269a5317e94d72d"},"cell_type":"code","source":"rec_num = 10\ncol = 'hitProductNames_src'\ns = train_df_r[col]\ncnt = s.value_counts(dropna=False)\nprint(len(cnt), cnt.nunique())\nprint(cnt.head(rec_num).append(cnt.tail(rec_num)).to_dict())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d752aba9e54c77f3f5a47339b6bce074f7ceb0d"},"cell_type":"code","source":"tvr = TfidfVectorizer(token_pattern=r'(?u)\\w+|[^\\w\\s]', strip_accents='unicode')\ntr_keyword = tvr.fit_transform(train_df_r[col])\ntokens = tvr.get_feature_names()\nprint(len(tokens))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c438c5951a781e85f038255e3c8ed216f3c4fbba"},"cell_type":"markdown","source":"## dates"},{"metadata":{"trusted":true,"_uuid":"1f7b19bdb88413ae32e90836d1a8e25e7e97ca73","scrolled":true},"cell_type":"code","source":"def encode_dates(df, date_encoders=None, date_data_types=None):\n    s = pd.to_datetime(df.date.astype(np.str))\n    s1 = pd.to_datetime(df.visitStartTime, unit='s')\n\n    df['visit_idle'] = (s1 - s).dt.days.astype(np.int8)\n    df['visit_year'] = (2018 - s1.dt.year).astype(np.int8)\n    df['visit_month'] = s1.dt.month.astype(np.int8)\n    df['visit_day'] = s1.dt.day.astype(np.int8)\n    df['visit_hour'] = s1.dt.hour.astype(np.int8)\n    df['visit_week'] = s1.dt.week.astype(np.int8)\n    df['visit_dayofweek'] = s1.dt.dayofweek.astype(np.int8)\n    df['visit_dayofyear'] = s1.dt.dayofyear.astype(np.int16)\n    df['visit_quarter'] = s1.dt.quarter.astype(np.int8)\n    df['visit_weekend'] = df['visit_dayofweek'].isin([5,6]).astype(np.int8)\n    \n    df['visit_hour_pv'] = df.visitStartTime // 3600\n    df['visit_day_pv'] = df.visit_hour_pv // 24\n    df,date_encoders,date_data_types = encode_obj(df, [f'visit_{tag}_pv' for tag in ['hour', 'day']], \n                                                  date_encoders, date_data_types, count_unique=False)\n    \n    return df.drop('date', axis=1),date_encoders,date_data_types","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"81ca3dbc8047a5c27f58883ef6a0f84cd49a7b37"},"cell_type":"code","source":"train_df_r,date_encoders,date_data_types = encode_dates(train_df_r)\ngc.collect()\nprint(f'train_df_r: {train_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3781cf6c268d2725a202f9e8f53f543e45e95ba","scrolled":false},"cell_type":"code","source":"s1 = train_df_r.visitId\ns2 = train_df_r.visitStartTime\nss = s2 - s1\nprint(ss.loc[ss>0].shape)\nprint(ss.quantile([0.994,0.995,0.996,0.997,0.998,0.999]))\nprint(ss.loc[ss>1200].shape, ss.loc[ss>1800].shape, ss.loc[ss>3600].shape, list(ss.loc[ss>3600]))\n\ntrain_df_r['visit_delay'] = (train_df_r.visitStartTime - train_df_r.visitId).astype(np.int16)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"134d3b94b0c64258e31fe4e55b64658750af4feb"},"cell_type":"markdown","source":"## save train_df_r"},{"metadata":{"trusted":true,"_uuid":"256789f6c4788042cf6b1212ea32e91a904a3697"},"cell_type":"code","source":"train_df_r.to_pickle('train_df_r', compression='gzip')\ndel train_df_r\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cbd3717add2e6c9965b8551ba31cc37eb3256ff7"},"cell_type":"markdown","source":"## process test"},{"metadata":{"trusted":true,"_uuid":"467a04a4b481418b72af85bac85582bba63860eb"},"cell_type":"code","source":"tr_df = load_df(sub_ids)\nprint(f'tr_df: {tr_df.shape}')\ngc.collect()\ntest_df_r = load_df(sub_ids, 'test')\nprint(f'test_df_r: {test_df_r.shape}')\ngc.collect()\ntest_df_r = tr_df.append(test_df_r, ignore_index=True, sort=False)\ndel tr_df\ngc.collect()\nprint(f'test_df_r: {test_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb389c44fa0e85f2342f1d4050224c0f39e6712b"},"cell_type":"code","source":"test_df_r = test_df_r.drop(still_cols, axis=1, errors='ignore')\ngc.collect()\nprint(f'test_df_r: {test_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e93ac6b42dc913650a2e5791e751f841182bb9a"},"cell_type":"code","source":"test_df_r = encode_user_pv(test_df_r)\ngc.collect()\nprint(f'test_df_r: {test_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12a3b3666211312898764e974b00d8562b78b11a"},"cell_type":"code","source":"col = 'trafficSource.keyword'\ntest_df_r[col] = test_df_r[col].str.lower()\nfor col in ['device.browser','trafficSource.adContent','trafficSource.keyword','trafficSource.referralPath','trafficSource.source']:\n    test_df_r[f'{col}_src'] = test_df_r[col].fillna('unknown')\n\ntest_df_r, cat_encoders, cat_data_types = encode_obj(test_df_r, obj_cols, cat_encoders, cat_data_types)\ngc.collect()\ntest_df_r = convert_data_type(test_df_r)\ngc.collect()\nprint(f'test_df_r: {test_df_r.shape}')\n\ntest_df_r, ref_encoders, ref_data_types = encode_ref_path(test_df_r, ref_encoders, ref_data_types)\ngc.collect()\ntest_df_r, src_encoders, src_data_types = encode_source(test_df_r, src_encoders, src_data_types)\ngc.collect()\nprint(f'test_df_r: {test_df_r.shape}')\n\ntest_df_r, date_encoders, date_data_types = encode_dates(test_df_r, date_encoders, date_data_types)\ngc.collect()\ntest_df_r['visit_delay'] = (test_df_r.visitStartTime - test_df_r.visitId).astype(np.int16)\ngc.collect()\nprint(f'test_df_r: {test_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7cb7e2e511b121b20b3b71665ea81e5298b620b6"},"cell_type":"markdown","source":"## save test_df_r"},{"metadata":{"trusted":true,"_uuid":"a57e8c634355df0fa0a52b502ae055e57e9357d2"},"cell_type":"code","source":"test_df_r.to_pickle('test_df_r', compression='gzip')\ndel test_df_r\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f8c6f68d9de1f51088903a76d381c1d49f03732"},"cell_type":"markdown","source":"# check"},{"metadata":{"trusted":true,"_uuid":"9b056f02ffee365518ac15532cc766fe1ce7c13a"},"cell_type":"code","source":"train_df_r = pd.read_pickle('train_df_r', compression='gzip')\ntest_df_r = pd.read_pickle('test_df_r', compression='gzip')\nprint(f'train_df_r: {train_df_r.shape}, test_df_r: {test_df_r.shape}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e596e1b0dc2a6e447904973317ab9383cb4e3c6"},"cell_type":"code","source":"cnt1 = train_df_r.visit_week.value_counts()\nprint(len(cnt1), cnt1.to_dict())\ncnt2 = test_df_r.visit_week.value_counts()\nprint(len(cnt2), cnt2.to_dict())\n\nx1 = cnt1.sort_index().index.values\ny1 = cnt1.sort_index().values\nx2 = cnt2.sort_index().index.values\ny2 = cnt2.sort_index().values\nplt.figure(figsize=(12,6))\nplt.plot(x1, y1)\nplt.plot(x2, y2)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b395a45d0019e28ca972edf3e505d65b869e0fdf"},"cell_type":"code","source":"s = train_df_r.dtypes\nss = list(s.loc[s != np.object].index)\nprint(len(ss))\nprint(ss)\nss = list(s.loc[s == np.object].index)\nprint(len(ss))\nprint(ss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb8da3ef3b62390f30ec765988efcd763625572b","scrolled":true},"cell_type":"code","source":"train_df_r.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"243821585adf3a7bf8fcf86daf05917932e9de60"},"cell_type":"code","source":"test_df_r.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"140e8961c9348ba3a94deb7624783e21f6a84e89"},"cell_type":"code","source":" ","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}