{"cells":[{"metadata":{"_uuid":"508335fc44f0c61a5a92f27400d180c9ac544014"},"cell_type":"markdown","source":"In this kernel, I checked scores of all 0 prediction at various evaluation date range. "},{"metadata":{"trusted":true,"_uuid":"814f9922233d87463069d6182ec43a658fb6d5eb","_kg_hide-input":true},"cell_type":"code","source":"%matplotlib inline\nimport matplotlib.pyplot as plt\n\nimport pandas as pd\nimport numpy as np\nimport json\nfrom pandas.io.json import json_normalize\n\nimport os\nimport gc\n\nfrom sklearn.metrics import mean_squared_error","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efa8e3ffb109cf377d1b0649ac9b1faba6882319"},"cell_type":"markdown","source":"## data preparation"},{"metadata":{"trusted":true,"_uuid":"012656a30ca03c413ebfb9304248cfe36a55ff74","_kg_hide-input":true},"cell_type":"code","source":"def load_df(csv_path, chunksize=100000):\n    features = ['date', 'fullVisitorId', 'totals_transactionRevenue']\n    JSON_COLS = ['totals']\n    print('Load {}'.format(csv_path))\n    df_reader = pd.read_csv(csv_path,\n                            converters={ column: json.loads for column in JSON_COLS },\n                            dtype={ 'date': str, 'fullVisitorId': str},\n                            usecols=['date', 'fullVisitorId', 'totals'], \n                            chunksize=chunksize)\n    res = pd.DataFrame()\n    for cidx, df in enumerate(df_reader):\n        df.reset_index(drop=True, inplace=True)\n        for col in JSON_COLS:\n            col_as_df = json_normalize(df[col])\n            col_as_df.columns = ['{}_{}'.format(col, subcol) for subcol in col_as_df.columns]\n            df = df.drop(col, axis=1).merge(col_as_df, right_index=True, left_index=True)\n        res = pd.concat([res, df[features]], axis=0).reset_index(drop=True)\n        del df\n        gc.collect()\n        print('{}: {}'.format(cidx + 1, res.shape))\n    return res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a927136c79c42bb2b9b801fb7f6201e65544a43c"},"cell_type":"code","source":"train = load_df('../input/train_v2.csv')\ntest = load_df('../input/test_v2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6555e1545e13ce9455b8464c80560c4b7ff9f76"},"cell_type":"code","source":"full_df = train.append(test).reset_index(drop=True)\nfull_df['date'] = pd.to_datetime(full_df['date'])\nfull_df.loc[:, full_df.columns.str.startswith('totals_')] = full_df.loc[:, full_df.columns.str.startswith('totals_')].astype(float).fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8afc6a4997a194450ec17dc523035ff31f59caf"},"cell_type":"code","source":"full_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d399b65d63b09681af97e85b1e26a9e052a57294"},"cell_type":"markdown","source":"## Extract target fullVisitorId at Each Validation Period"},{"metadata":{"_uuid":"389de35651e17bcf3e17f0360203de89dce52d9b"},"cell_type":"markdown","source":"Processing to extract the fulVisitorId having the log from 7 months ago to 1.5 months before the evaluation start date as the evaluation target."},{"metadata":{"trusted":true,"_uuid":"fc75c1d4112d7c1df16f2621237bbdb62610fce6"},"cell_type":"code","source":"def get_target_fullvisitorid(full_df, target_datestart):\n    target_fullvisitorid = pd.DataFrame(\n        full_df.loc[\n            (full_df['date'] >= pd.to_datetime(target_datestart)-pd.DateOffset(214)) &  \n            (full_df['date'] < pd.to_datetime(target_datestart)-pd.DateOffset(45)), \n            'fullVisitorId'\n        ].unique(), \n        columns=['fullVisitorId']).reset_index(drop=True)\n    return target_fullvisitorid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f2633e91b5172527649a04ba6d20a1205d03f75"},"cell_type":"markdown","source":"Checking whether it properly extract fullVisitorIds in test set."},{"metadata":{"scrolled":true,"trusted":true,"_uuid":"129edc14abaf5936e1554d58ac061a0e0c943e89"},"cell_type":"code","source":"test_ids = get_target_fullvisitorid(full_df, '2018-12-01')\nss = pd.read_csv('../input/sample_submission_v2.csv', dtype={'fullVisitorId': str},)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e24bcbe9541ea7f5f901d1f84aafba9ec5322e8e"},"cell_type":"code","source":"print((~test_ids['fullVisitorId'].isin(ss['fullVisitorId'])).sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4730efdd37374815cb751946fdaf3d84a4fadae5"},"cell_type":"code","source":"print((~ss['fullVisitorId'].isin(test_ids['fullVisitorId'])).sum())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4a820264da7bfea1d45e26c7cbc66bf5e3a019c"},"cell_type":"markdown","source":"It seems to extract fullVisitorIds without omission."},{"metadata":{"_uuid":"7845725575372e83a891d09fa70c8f6a4aa6915c"},"cell_type":"markdown","source":"## Making Gound Truth and Checking All 0 Prediction Score at Each Validation Period"},{"metadata":{"_uuid":"3d6f31d613a869e1245c305882fb6318cf0676a5"},"cell_type":"markdown","source":"The verification period is set to 2 months (61days) according to competition task."},{"metadata":{"trusted":true,"_uuid":"26a5e5e230ee16ad3cd68b7f8d6b4ad33c700500"},"cell_type":"code","source":"def make_groundtruth(target_datestart):\n    target_fullvisitorid = get_target_fullvisitorid(full_df, target_datestart)\n    \n    date_range_for_groundtruth = [pd.to_datetime(target_datestart), pd.to_datetime(target_datestart)+pd.DateOffset(61)]\n    \n    exist_users_groundtruth = full_df.loc[\n        (full_df['date'] >= date_range_for_groundtruth[0]) &  \n        (full_df['date'] < date_range_for_groundtruth[1]) &\n        (full_df['fullVisitorId'].isin(target_fullvisitorid['fullVisitorId'])), \n        ['fullVisitorId', 'totals_transactionRevenue']\n    ].groupby('fullVisitorId')['totals_transactionRevenue'].sum()\\\n    .to_frame(name='LogSumRevenue').apply(np.log1p)\n    full_groundtruth = target_fullvisitorid.merge(exist_users_groundtruth.reset_index(), on='fullVisitorId', how='left').set_index('fullVisitorId').fillna(0)\n        \n    return date_range_for_groundtruth, full_groundtruth","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5622a1b45c3a67edbf376a0dc4be0006815997c"},"cell_type":"code","source":"for target_datestart in ['2017-12-01', '2018-01-01', '2018-02-01', \n                         '2018-03-01', '2018-04-01', '2018-05-01', \n                         '2018-06-01', '2018-07-01', '2018-08-01']:\n    date_range_for_groundtruth, groundtruth = make_groundtruth(target_datestart)\n    all0_pred = groundtruth.assign(pred=0)[['pred']]\n    score = np.sqrt(mean_squared_error(groundtruth['LogSumRevenue'], all0_pred['pred']))\n    print('validatin date range:{0} to {1} num of fullVisitorIds:{2} score:{3:.5f}'\\\n              .format(str(date_range_for_groundtruth[0].date()), str(date_range_for_groundtruth[1].date()), groundtruth.shape[0], score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"30e56fb82fb2fbcb4b19ba6cc183cd58e3ddea4d"},"cell_type":"markdown","source":"For most of time range,  all 0 prediction appears to be scored around 0.15-0.35."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}