{"cells":[{"metadata":{"_uuid":"c812c6be2929c2cb79007f388e4453983cafe192"},"cell_type":"markdown","source":"This competition looks ripe for attacking with libffm regression\n\nI stole the basic data munging \n\nIt is a pain to munge the data into libffm format so this script will do it for you!\n\nYou can go to https://www.csie.ntu.edu.tw/~cjlin/libffm/ to get the regression code!\n\nAll the best\n\nScirpus"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport numpy as np \nimport pandas as pd \nimport json\nfrom pandas.io.json import json_normalize\nimport seaborn as sns \nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import TimeSeriesSplit, KFold\nfrom sklearn.metrics import mean_squared_error\nimport gc\nimport matplotlib.pyplot as plt\nimport seaborn as sns\npd.options.mode.chained_assignment = None\npd.options.display.max_columns = 999\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"PATH=\"../input/\"\n \ncols_to_parse = ['device', 'geoNetwork', 'totals', 'trafficSource']\n\ndef read_parse_dataframe(file_name):\n    #full path for the data file\n    path = PATH + file_name\n    #read the data file, convert the columns in the list of columns to parse using json loader,\n    #convert the `fullVisitorId` field as a string\n    data_df = pd.read_csv(path, \n        converters={column: json.loads for column in cols_to_parse}, \n        dtype={'fullVisitorId': 'str'})\n    #parse the json-type columns\n    for col in cols_to_parse:\n        #each column became a dataset, with the columns the fields of the Json type object\n        json_col_df = json_normalize(data_df[col])\n        json_col_df.columns = [f\"{col}.{sub_col}\" for sub_col in json_col_df.columns]\n        #we drop the object column processed and we add the columns created from the json fields\n        data_df = data_df.drop(col, axis=1).merge(json_col_df, right_index=True, left_index=True)\n\n    return data_df\n    \ndef process_date_time(data_df):\n    data_df['date'] = data_df['date'].astype(str)\n    data_df[\"date\"] = data_df[\"date\"].apply(lambda x : x[:4] + \"-\" + x[4:6] + \"-\" + x[6:])\n    data_df[\"date\"] = pd.to_datetime(data_df[\"date\"])   \n    data_df[\"year\"] = data_df['date'].dt.year\n    data_df[\"month\"] = data_df['date'].dt.month\n    data_df[\"day\"] = data_df['date'].dt.day\n    data_df[\"weekday\"] = data_df['date'].dt.weekday\n    data_df['weekofyear'] = data_df['date'].dt.weekofyear\n    data_df['month.unique.user.count'] = data_df.groupby('month')['fullVisitorId'].transform('nunique')\n    data_df['day.unique.user.count'] = data_df.groupby('day')['fullVisitorId'].transform('nunique')\n    data_df['weekday.unique.user.count'] = data_df.groupby('weekday')['fullVisitorId'].transform('nunique')\n    return data_df\n\ndef process_format(data_df):\n    for col in ['visitNumber', 'totals.hits', 'totals.pageviews']:\n        data_df[col] = data_df[col].astype(float)\n    data_df['trafficSource.adwordsClickInfo.isVideoAd'].fillna(True, inplace=True)\n    data_df['trafficSource.isTrueDirect'].fillna(False, inplace=True)\n    return data_df\n    \ndef process_device(data_df):\n    data_df['browser.category'] = data_df['device.browser'] + '.' + data_df['device.deviceCategory']\n    data_df['browser.os'] = data_df['device.browser'] + '.' + data_df['device.operatingSystem']\n    return data_df\n\ndef process_totals(data_df):\n    data_df['visitNumber'] = (data_df['visitNumber'])\n    data_df['totals.hits'] = (data_df['totals.hits'])\n    data_df['totals.pageviews'] = (data_df['totals.pageviews'].fillna(0))\n    data_df['mean.hits.per.day'] = data_df.groupby(['day'])['totals.hits'].transform('mean')\n    data_df['sum.hits.per.day'] = data_df.groupby(['day'])['totals.hits'].transform('sum')\n    data_df['max.hits.per.day'] = data_df.groupby(['day'])['totals.hits'].transform('max')\n    data_df['min.hits.per.day'] = data_df.groupby(['day'])['totals.hits'].transform('min')\n    data_df['var.hits.per.day'] = data_df.groupby(['day'])['totals.hits'].transform('var')\n    data_df['mean.pageviews.per.day'] = data_df.groupby(['day'])['totals.pageviews'].transform('mean')\n    data_df['sum.pageviews.per.day'] = data_df.groupby(['day'])['totals.pageviews'].transform('sum')\n    data_df['max.pageviews.per.day'] = data_df.groupby(['day'])['totals.pageviews'].transform('max')\n    data_df['min.pageviews.per.day'] = data_df.groupby(['day'])['totals.pageviews'].transform('min')    \n    return data_df\n\ndef process_geo_network(data_df):\n    data_df['sum.pageviews.per.network.domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('sum')\n    data_df['count.pageviews.per.network.domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('count')\n    data_df['mean.pageviews.per.network.domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.pageviews'].transform('mean')\n    data_df['sum.hits.per.network.domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('sum')\n    data_df['count.hits.per.network.domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('count')\n    data_df['mean.hits.per.network.domain'] = data_df.groupby('geoNetwork.networkDomain')['totals.hits'].transform('mean')\n    return data_df\n\ndef process_traffic_source(data_df):\n    data_df['source.country'] = data_df['trafficSource.source'] + '.' + data_df['geoNetwork.country']\n    data_df['campaign.medium'] = data_df['trafficSource.campaign'] + '.' + data_df['trafficSource.medium']\n    data_df['medium.hits.mean'] = data_df.groupby(['trafficSource.medium'])['totals.hits'].transform('mean')\n    data_df['medium.hits.max'] = data_df.groupby(['trafficSource.medium'])['totals.hits'].transform('max')\n    data_df['medium.hits.min'] = data_df.groupby(['trafficSource.medium'])['totals.hits'].transform('min')\n    data_df['medium.hits.sum'] = data_df.groupby(['trafficSource.medium'])['totals.hits'].transform('sum')\n    return data_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"545968b47460691e9b0962ed42f9c9313f7b041f"},"cell_type":"code","source":"train_df = read_parse_dataframe('train.csv')\ntest_df = read_parse_dataframe('test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c9bdcdc9d295ad16733e6f70238c9b6406d3d4f"},"cell_type":"code","source":"train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b76569d2e37f0b4be43e34f0853c89a26265410d"},"cell_type":"code","source":"cols_to_drop = [col for col in train_df.columns if train_df[col].nunique(dropna=False) == 1]\ntrain_df.drop(cols_to_drop, axis=1, inplace=True)\ntest_df.drop([col for col in cols_to_drop if col in test_df.columns], axis=1, inplace=True)\ntrain_df.drop(['trafficSource.campaignCode'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"582818efac797a83da5f91ae29c4971dda63f1e1"},"cell_type":"code","source":"train_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].astype(float)\ntrain_df['totals.transactionRevenue'] = train_df['totals.transactionRevenue'].fillna(0)\ntrain_df['totals.transactionRevenue'] = np.log1p(train_df['totals.transactionRevenue'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"099c324b0d3a1cb543db5d55c9219e054cd10ea6"},"cell_type":"code","source":"train_df = process_date_time(train_df)\ntrain_df = process_format(train_df)\ntrain_df = process_device(train_df)\ntrain_df = process_totals(train_df)\ntrain_df = process_geo_network(train_df)\ntrain_df = process_traffic_source(train_df)\n\ntest_df = process_date_time(test_df)\ntest_df = process_format(test_df)\ntest_df = process_device(test_df)\ntest_df = process_totals(test_df)\ntest_df = process_geo_network(test_df)\ntest_df = process_traffic_source(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58a0916dec97ef0baff49062c4c817221c2ac5ab"},"cell_type":"code","source":"num_cols = ['month.unique.user.count', 'day.unique.user.count', 'weekday.unique.user.count',\n            'visitNumber', 'totals.hits', 'totals.pageviews', \n            'mean.hits.per.day', 'sum.hits.per.day', 'min.hits.per.day', 'max.hits.per.day', 'var.hits.per.day',\n            'mean.pageviews.per.day', 'sum.pageviews.per.day', 'min.pageviews.per.day', 'max.pageviews.per.day',\n            'sum.pageviews.per.network.domain', 'count.pageviews.per.network.domain', 'mean.pageviews.per.network.domain',\n            'sum.hits.per.network.domain', 'count.hits.per.network.domain', 'mean.hits.per.network.domain',\n            'medium.hits.mean','medium.hits.min','medium.hits.max','medium.hits.sum']\n                \nnot_used_cols = [\"visitNumber\", \"date\", \"fullVisitorId\", \"sessionId\", \n                 \"visitId\", \"visitStartTime\", 'totals.transactionRevenue', 'trafficSource.referralPath']\ncat_cols = [col for col in train_df.columns if col not in num_cols and col not in not_used_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"86c8c0c866ac68127535e6f8eed179bf47e5d753"},"cell_type":"code","source":"for col in num_cols:\n    train_df[col] = np.log1p((train_df[col].values))\n    test_df[col] = np.log1p((test_df[col].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c43e8c7b14822a051111aed0fa50388e6db2da1c"},"cell_type":"code","source":"x = pd.concat([train_df,test_df],sort=False)\nx = x.reset_index(drop=True)\nfor col in num_cols:\n    x.loc[:,col] = pd.cut(x[col], 50,labels=False)\ntest_df = x.loc[train_df.shape[0]:].copy().reset_index(drop=True)\ntrain_df = x.loc[:train_df.shape[0]].copy().reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4b8ef2425e55ee8fda9b8bc07bd95aa375cd3b9"},"cell_type":"code","source":"for col in cat_cols:\n    lbl = LabelEncoder()\n    lbl.fit(list(train_df[col].values.astype('str')) + list(test_df[col].values.astype('str')))\n    train_df[col] = lbl.transform(list(train_df[col].values.astype('str')))\n    test_df[col] = lbl.transform(list(test_df[col].values.astype('str')))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f812476dff3002252b6cba6cec57f610c6b8541"},"cell_type":"code","source":"train_df.fillna(0,inplace=True,axis=1)\ntest_df.fillna(0,inplace=True,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8f3984cc098e1a648b1173ce532326258b6d577"},"cell_type":"code","source":"test_df['totals.transactionRevenue'] = 0.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3c8e9f4b1ec69327a380f3bb8e5bbec6761bacb"},"cell_type":"code","source":"features = num_cols+cat_cols\ncategories = features[:]\nnumerics = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc0b40c3477e2543aec751e2319b645c9858a22b"},"cell_type":"code","source":"currentcode = len(numerics)\ncatdict = {}\ncatcodes = {}\nfor x in numerics:\n    catdict[x] = 0\nfor x in categories:\n    catdict[x] = 1\n\nnoofrows = train_df.shape[0]\nnoofcolumns = len(features)\nwith open(\"alltrainffm.txt\", \"w\") as text_file:\n    for n, r in enumerate(range(noofrows)):\n        if((n%100000)==0):\n            print('Row',n)\n        datastring = \"\"\n        datarow = train_df.iloc[r].to_dict()\n        datastring += str(float(datarow['totals.transactionRevenue']))\n\n\n        for i, x in enumerate(catdict.keys()):\n            if(catdict[x]==0):\n                datastring = datastring + \" \"+str(i)+\":\"+ str(i)+\":\"+ str(datarow[x])\n            else:\n                if(x not in catcodes):\n                    catcodes[x] = {}\n                    currentcode +=1\n                    catcodes[x][datarow[x]] = currentcode\n                elif(datarow[x] not in catcodes[x]):\n                    currentcode +=1\n                    catcodes[x][datarow[x]] = currentcode\n\n                code = catcodes[x][datarow[x]]\n                datastring = datastring + \" \"+str(i)+\":\"+ str(int(code))+\":1\"\n        datastring += '\\n'\n        text_file.write(datastring)\n        \nnoofrows = test_df.shape[0]\nnoofcolumns = len(features)\nwith open(\"alltestffm.txt\", \"w\") as text_file:\n    for n, r in enumerate(range(noofrows)):\n        if((n%100000)==0):\n            print('Row',n)\n        datastring = \"\"\n        datarow = test_df.iloc[r].to_dict()\n        datastring += str(float(datarow['totals.transactionRevenue']))\n\n\n        for i, x in enumerate(catdict.keys()):\n            if(catdict[x]==0):\n                datastring = datastring + \" \"+str(i)+\":\"+ str(i)+\":\"+ str(datarow[x])\n            else:\n                if(x not in catcodes):\n                    catcodes[x] = {}\n                    currentcode +=1\n                    catcodes[x][datarow[x]] = currentcode\n                elif(datarow[x] not in catcodes[x]):\n                    currentcode +=1\n                    catcodes[x][datarow[x]] = currentcode\n\n                code = catcodes[x][datarow[x]]\n                datastring = datastring + \" \"+str(i)+\":\"+ str(int(code))+\":1\"\n        datastring += '\\n'\n        text_file.write(datastring)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}