{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nfrom datetime import datetime, timedelta\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"tr=pd.read_csv('/kaggle/input/save-my-data/train-flattened.csv')\ntest=pd.read_csv('/kaggle/input/save-my-data/test-flattened.csv',nrows = 2000)\nsub=pd.read_csv('/kaggle/input/save-my-data/submit-flattened.csv',nrows=2000)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### change the tpye for engineering"},{"metadata":{"trusted":true},"cell_type":"code","source":"tr[\"date\"] = pd.to_datetime(tr[\"date\"], infer_datetime_format=True, format=\"%Y%m%d\")\ntr['totals_pageviews'] = tr['totals_pageviews'].astype(float)\ntr['totals_newVisits'] = tr['totals_newVisits'].astype(float)\ntr['trafficSource_isTrueDirect'] = tr['trafficSource_isTrueDirect'].astype(bool)\ntr['totals_hits'] = tr['totals_hits'].astype(float)\ntr['device_isMobile'] = tr['device_isMobile'].astype(bool)\ntr['totals_timeOnSite'] = tr['totals_timeOnSite'].astype(float)\ntr['totals_transactions'] = tr['totals_transactions'].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test[\"date\"] = pd.to_datetime(test[\"date\"], infer_datetime_format=True, format=\"%Y%m%d\")\ntest['totals_pageviews'] = test['totals_pageviews'].astype(float)\ntest['totals_newVisits'] = test['totals_newVisits'].astype(float)\ntest['trafficSource_isTrueDirect'] = test['trafficSource_isTrueDirect'].astype(bool)\ntest['totals_hits'] = test['totals_hits'].astype(float)\ntest['device_isMobile'] = test['device_isMobile'].astype(bool)\ntest['totals_timeOnSite'] = test['totals_timeOnSite'].astype(float)\ntest['totals_transactions'] = test['totals_transactions'].astype(float)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#replace unknown value with 0\nunknownvalues=['(not set)','not available in demo dataset','(not provided)','unknown.unknown','/','Not Socially Engaged']\nfor unknownvalue in unknownvalues:    \n    tr.replace(unknownvalue, np.nan, inplace=True)\n    test.replace(unknownvalue, np.nan, inplace=True)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['totals_transactionRevenue'].replace(np.nan, 0,inplace=True)\ntest['totals_transactionRevenue'].replace(np.nan, 0,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_constant=[i for i in tr.columns if tr[i].nunique()>1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr=tr[non_constant]\ntest=test[tr.columns]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"    test         gap          target\n    168 days    46days        62days\n    \n    train\n    276\n    168  46  62"},{"metadata":{"trusted":true},"cell_type":"code","source":"timedelta(days=168).days","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr['date'].max()-tr['date'].min()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"df = tr"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['fullVisitorId'] = df['fullVisitorId'].astype(str)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_groupby_target(df,k=0): \n    df['fullVisitorId'] = df['fullVisitorId'].astype(str)\n    start_date=(df['date'].min()+ timedelta(days=k))\n    end_date=(df['date'].min() + timedelta(days=168+k))\n    mask =  (df['date'] > start_date) & (df['date'] <= end_date)\n    tf=df.loc[mask]\n    tfg = tf.groupby('fullVisitorId').agg({\n            'geoNetwork_networkDomain': {'networkDomain': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'geoNetwork_city': {'city':lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'device_operatingSystem': {'operatingSystem': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'geoNetwork_metro': {'metro': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'geoNetwork_region': {'region':lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'channelGrouping': {'channelGrouping':lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'trafficSource_referralPath': {'referralPath': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'geoNetwork_country': {'country': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'trafficSource_source': {'source': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'trafficSource_medium': {'medium': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'trafficSource_keyword': {'keyword': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'device_browser':  {'browser': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'trafficSource_adwordsClickInfo.gclId': {'gclId': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'device_deviceCategory': {'deviceCategory': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'geoNetwork_continent': {'continent': lambda x: -999 if pd.isnull(x.dropna().max()) else 1},\n            'totals_timeOnSite': {'timeOnSite_max': lambda x: x.dropna().max(),\n                                  'timeOnSite_min': lambda x: x.dropna().min(), \n                                  'timeOnSite_mean': lambda x: x.dropna().mean(),\n                                  'timeOnSite_sum': lambda x: x.dropna().sum()},\n            'totals_pageviews': {'pageviews_max': lambda x: x.dropna().max(),\n                                 'pageviews_min': lambda x: x.dropna().min(),\n                                 'pageviews_mean': lambda x: x.dropna().mean(),\n                                 'pageviews_sum': lambda x: x.dropna().sum()},\n            'totals_hits': {'hits_max': lambda x: x.dropna().max(), \n                            'hits_min': lambda x: x.dropna().min(),\n                            'hits_mean': lambda x: x.dropna().mean(),\n                            'hits_sum': lambda x: x.dropna().sum()},\n            'visitStartTime': {'visitStartTime_counts': lambda x: x.dropna().count()},\n            'totals_sessionQualityDim': {'sessionQualityDim': lambda x: x.dropna().max()},\n            'device_isMobile': {'isMobile': lambda x: x.dropna().max()},\n            'visitNumber': {'visitNumber_max' : lambda x: x.dropna().max()}, \n            'totals_totalTransactionRevenue':  {'totalTransactionRevenue_sum':  lambda x:x.dropna().sum()},\n            'totals_transactions' : {'transactions' : lambda x:x.dropna().sum()},\n            'date':{\"session\":lambda x :(x.dropna().max()- x.dropna().min()).days}})\n    \n    target_strat=(start_date+ timedelta(days=214))\n    target_end=(target_strat + timedelta(days=62))\n    mask1 =  (df['date'] >= target_strat) & (df['date'] <= target_end)\n    tf2=df.loc[mask1]\n    tf3=tf2.groupby('fullVisitorId').agg({'totals_totalTransactionRevenue':{'target': lambda x: x.dropna().sum()}})\n    tfg.columns=tfg.columns.droplevel()\n    tf3.columns=['target']\n    tfg['fullVisitorId'] = tfg.index\n    tf3['fullVisitorId'] = tf3.index\n    \n    tfg['target']=0\n    tfg['return']=0\n    for i in tfg['fullVisitorId']:\n        if i in tf3['fullVisitorId']:\n            tfg.loc[i,'target']=tf3.loc[i,'target']\n            tfg.loc[i,'return']=1\n    \n    return tfg\n    \n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_0 = get_groupby_target(tr,0)\n%time\ntrain_1 = get_groupby_target(tr,62)\n%time\ntrain_2 = get_groupby_target(tr,124)\n%time\ntrain_3 = get_groupby_target(tr,186)\n%time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_train = pd.concat([train_0,train_1,train_2,train_3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_train.to_csv(\"my_train_fisrt_part.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_train['return'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"use_train[use_train['return']==1]","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}