{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/codlife/pre-processing-for-huge-train-data-with-chunksize\n# https://www.kaggle.com/usmanabbas/flatten-columns-remove-columns-before-normalizing\nimport os\nimport json\nimport numpy as np\nfrom numpy import loadtxt\nimport pandas as pd\nfrom pandas.io.json import json_normalize\nfrom ast import literal_eval\nimport gc\nimport xgboost as xgb\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import GridSearchCV\ngc.enable()\ndef load_df(csv_path):\n    #csv contains nested data as JSON\n    JSON_COLUMNS = ['device', 'geoNetwork', 'totals', 'trafficSource']\n    ans = pd.DataFrame()\n    dfs = pd.read_csv(csv_path, sep=',',\n                     converters={column: json.loads for column in JSON_COLUMNS}, \n                     dtype={'fullVisitorId': 'str','visitId': 'str'}, \n                    chunksize = 100000)\n    chunk_count=0 #limit chunks\n    #Ignored for now \n    #due to dimension ('customDimensions','hits')\n    bad_columns = ['customDimensions','hits','date','trafficSource.campaign','socialEngagementType','trafficSource.adwordsClickInfo.gclId']\n    #start loading chunks\n    for df in dfs:\n        gc.collect()\n        df.reset_index(drop = True,inplace = True)\n        #unnest JSON_COLUMNS\n        for column in JSON_COLUMNS:\n            column_as_df = json_normalize(df[column])\n            column_as_df.columns = [f\"{column}.{subcolumn}\" for subcolumn in column_as_df.columns]\n            df = df.drop(column, axis=1).merge(column_as_df, right_index=True, left_index=True)\n        #drop columns that contains irrelevant data like 'not available in demo dataset'\n        droped_cols = []\n        for col in df.columns:\n            if 'not available in demo dataset'in df[col].unique() :\n                    droped_cols = np.append(droped_cols,col)\n        df = df.drop(axis='1', columns = droped_cols)\n        #print(f'Droped {droped_cols}')\n        #change type of columns for better handling\n        df['totals.bounces']=df['totals.bounces'].apply(lambda x : x == '1')\n        df['totals.newVisits']=df['totals.newVisits'].apply(lambda x : x == '1')\n        df['trafficSource.isTrueDirect']=df['trafficSource.isTrueDirect'].apply(lambda x : x == True)\n        df['trafficSource.adwordsClickInfo.isVideoAd']=df['trafficSource.adwordsClickInfo.isVideoAd'].apply(lambda x : x == True)\n        #print(f\"Droping columns {bad_columns}\")\n        df = df.drop(axis='1', columns = bad_columns)\n        ans = pd.concat([ans, df], axis = 0,sort=False).reset_index(drop = True)\n        print(f\"Shape = Shape: {ans.shape}\")\n        chunk_count = chunk_count + 1 \n        if(chunk_count == 10):\n            break\n    return ans","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"train = load_df('../input/train_v2.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93653fadada70a89c6e3f9867d471e1150fd5cd9","scrolled":false},"cell_type":"code","source":"train.describe(include=\"object\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46c4428c8e370230668419c254c2de2c9bca19fb"},"cell_type":"code","source":"train.describe(include=\"bool\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52b879dcd0c8b97dc9401e0c1d6fe87aeb738eea","scrolled":true},"cell_type":"code","source":"pd.options.display.float_format = '{:.2f}'.format\ntrain.describe(include=\"int64\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a08b0101f6392d067ee6ec6bcc1b71b20323456"},"cell_type":"code","source":"def date_preProcessing(df):\n    df['date'] = pd.to_datetime(df['visitStartTime'], unit='s')\n    df['month'] = df['date'].dt.month\n    df['day'] = df['date'].dt.day\n    df['hour'] = df['date'].dt.hour\n    df['dayOfWeek'] = train['date'].dt.dayofweek\n    df['quarter'] = train['date'].dt.quarter\n    df['weekofyear'] = train['date'].dt.weekofyear\n    print(\"Droping column ['date','visitStartTime']\")\n    df = df.drop(axis='1', columns = ['date','visitStartTime'])\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d74cbfc57ff22b7cf94797965dc4501c15624d36"},"cell_type":"code","source":"train = date_preProcessing(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdb06571b26fc0e152e550bcc3731528e181a08a"},"cell_type":"code","source":"target = np.log1p(train['totals.transactionRevenue'].fillna(0).astype(float))\nids = train[['fullVisitorId']].copy()\nfeatures = train.drop(['fullVisitorId', 'visitId','totals.transactionRevenue'], axis=1, inplace=False)\nfor cols in features.columns:\n    if features[cols].dtype == 'object':\n        lbl = preprocessing.LabelEncoder()\n        lbl.fit(list(features[cols].values))\n        features[cols] = lbl.transform(list(features[cols].values))\nmodel = xgb.XGBRegressor()\nparameters = {'nthread':[4], \n              'objective':['reg:linear'],\n              'min_child_weight':[0.5,1,10],\n               'max_depth':[3,5,10],\n              'learning_rate': [.01, 0.05, .1], \n              'max_depth': [5, 6, 7],\n             'n_estimators': [500,100,1000]}\nxgb_grid = GridSearchCV(model,\n                        parameters,\n                        cv = 2,\n                        n_jobs = 1,\n                        verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76fb9a3d37b1bfa9d5ad4663fb046561b431e9ae"},"cell_type":"code","source":"xgb_grid.fit(features,target)\nprint(f\"Best R2 Score {xgb_grid.best_score_}\")\nprint(f\"Best Params {xgb_grid.best_params_}\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}