{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport json\nimport bq_helper\nfrom pandas.io.json import json_normalize\nimport seaborn as sns \nimport lightgbm as lgb\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import TimeSeriesSplit, KFold,GroupKFold\nfrom sklearn.metrics import mean_squared_error\nimport functools\nfrom multiprocessing import Pool\nimport logging\nimport gc\nimport logging\nimport matplotlib.pyplot as plt\nimport time\nfrom line_profiler import LineProfiler\nfrom sklearn.metrics import mean_squared_error\nfrom scipy.stats import stats\nimport os\nimport re\nprint(os.listdir(\"../input\"))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def line_profiling(func,arg):\n    lp=LineProfiler()\n    lp_wrapper=lp(func)\n    lp_wrapper(arg)\n    lp.print_stats()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"802cc6cc94c1bf22c8168b5f0fd671c1af80d0ad"},"cell_type":"code","source":"def process_date_time(data_df):\n    \n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month_unique_user_count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day_unique_user_count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday_unique_user_count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    data_df['weekofyear_unique_user_count'] = data_df.groupby('weekofyear')['fullVisitorId'].transform('nunique')\n    return data_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"71409f93edb0a0fc34efb519b5bc6cc1ee5cee09"},"cell_type":"code","source":"def process_format(data_df):\n\n    for col in ['visitNumber', 'totals.hits', 'totals.pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource.isTrueDirect'].fillna(False, inplace=True)\n    \n    return data_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"64789b5e7cb91985e12940aeae704b01da372fad"},"cell_type":"code","source":"def process_totals(data_df):\n    #data_df['visitNumber'] = data_df['visitNumber']\n    data_df['visits_id_sum'] = data_df.groupby(['fullVisitorId'])['visitNumber'].transform('sum')\n    data_df['visits_id_min'] =  data_df.groupby(['fullVisitorId'])['visitNumber'].transform('min')\n    data_df['visits_id_max'] = data_df.groupby(['fullVisitorId'])['visitNumber'].transform('max')\n    data_df['visits_id_mean'] = data_df.groupby(['fullVisitorId'])['visitNumber'].transform('mean')\n    data_df['visits_id_nunique'] = data_df.groupby('visitNumber')['fullVisitorId'].transform('nunique')\n    #data_df['totals_hits'] = data_df['totals_hits']\n    data_df['hits_id_sum'] = data_df.groupby(['fullVisitorId'])['totals.hits'].transform('sum')\n    data_df['hits_id_cnt'] = data_df.groupby(['fullVisitorId'])['totals.hits'].transform('count')\n    data_df['totals.pageviews'] = data_df['totals.pageviews'].fillna(0)\n    data_df['pageviews_id_sum'] = data_df.groupby(['fullVisitorId'])['totals.pageviews'].transform('sum')\n    data_df['pageviews_id_cnt'] = data_df.groupby(['fullVisitorId'])['totals.pageviews'].transform('count')\n    data_df['mean_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('mean')\n    data_df['sum_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('sum')\n    data_df['max_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('max')\n    data_df['min_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('min')\n    data_df['var_hits_per_day'] = data_df.groupby(['day'])['totals.hits'].transform('var')\n    data_df['mean_pageviews_per_day'] = data_df.groupby(['day'])['totals.pageviews'].transform('mean')\n    data_df['sum_pageviews_per_day'] = data_df.groupby(['day'])['totals.pageviews'].transform('sum')\n    data_df['max_pageviews_per_day'] = data_df.groupby(['day'])['totals.pageviews'].transform('max')\n    data_df['min_pageviews_per_day'] = data_df.groupby(['day'])['totals.pageviews'].transform('min')   \n    \n    return data_df\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e741d08f11e9869c180a0def780bfa3a52b5cf8"},"cell_type":"code","source":"train_df=pd.read_hdf('../input/load-data-faster-with-hdf/train_df.h5')\ntest_df=pd.read_hdf('../input/load-data-faster-with-hdf/test_df.h5')\nprint('train loaded')\nprint('test_loaded')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"57468bdf9aab1d5f5555aeb90e547a44b5e7aac2"},"cell_type":"markdown","source":"**<p> .transform() is almost 87 of the time in process_date_time function</p>**"},{"metadata":{"trusted":true,"_uuid":"7d33ccb83f1c4270cb5b811b14b951365b3d70e0"},"cell_type":"code","source":"line_profiling(process_date_time,train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e63bf3165be6ce8f9c1f1d27d6601423adcbef9"},"cell_type":"code","source":"line_profiling(process_format,train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fdddb4b0123b13e0ca21bf7808e90622d25d610"},"cell_type":"code","source":"train_df=process_format(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3203607b528bb88d4aea06de94043aab034ab966"},"cell_type":"code","source":"train_df=process_totals(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"a1b7c7050f4f619f9c617caf0ae99ad627692ded"},"cell_type":"code","source":"line_profiling(process_totals,train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26fd6fb06605d24ed53acffea132aacb184768c2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}