{"cells":[{"metadata":{},"cell_type":"markdown","source":"XGBoost with Half and Half implementation"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport gc\n\nimport os\nimport random\nimport math\nimport psutil\nimport pickle\n\nfrom sklearn.model_selection import train_test_split,KFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom IPython.display import FileLink\nimport xgboost as xgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Importing only training data to save on RAM"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#%%time\n\nweather_train = pd.read_csv(\"../input/ashrae-energy-prediction/weather_train.csv\")#, parse_dates=['timestamp'], dtype=weather_dtype)\nmetadata = pd.read_csv(\"../input/ashrae-energy-prediction/building_metadata.csv\")#, dtype=metadata_dtype)\ntrain = pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")#, parse_dates=['timestamp'], dtype=train_dtype)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%%time\ntrain = pd.merge(train,metadata,on='building_id',how='left')\nprint (\"Training Data+Metadata Shape {}\".format(train.shape))\n\ngc.collect()\ntrain = pd.merge(train,weather_train,on=['site_id','timestamp'],how='left')\nprint (\"Training Data+Metadata+Weather Shape {}\".format(train.shape))\ndel weather_train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\ntrain[\"primary_use\"] = le.fit_transform(train[\"primary_use\"])\ntrain['meter_reading'] = np.log1p(train['meter_reading'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using Arjanso's memory reducing function to save on memory."},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65\ndef reduce_mem_usage(df):\n    start_mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage of properties dataframe is :\",start_mem_usg,\" MB\")\n    NAlist = [] # Keeps track of columns that have missing values filled in. \n    for col in df.columns:\n        if df[col].dtype != object:  # Exclude strings            \n            # Print current column type\n#             print(\"******************************\")\n            print(\"Column: \",col)\n            #print(\"dtype before: \",df[col].dtype)            \n            # make variables for Int, max and min\n            IsInt = False\n            mx = df[col].max()\n            mn = df[col].min()\n#             print(\"min for this col: \",mn)\n#             print(\"max for this col: \",mx)\n            # Integer does not support NA, therefore, NA needs to be filled\n            if not np.isfinite(df[col]).all(): \n                NAlist.append(col)\n                df[col].fillna(mn-1,inplace=True)  \n                   \n            # test if column can be converted to an integer\n            asint = df[col].fillna(0).astype(np.int64)\n            result = (df[col] - asint)\n            result = result.sum()\n            if result > -0.01 and result < 0.01:\n                IsInt = True            \n            # Make Integer/unsigned Integer datatypes\n            if IsInt:\n                if mn >= 0:\n                    if mx < 255:\n                        df[col] = df[col].astype(np.uint8)\n                    elif mx < 65535:\n                        df[col] = df[col].astype(np.uint16)\n                    elif mx < 4294967295:\n                        df[col] = df[col].astype(np.uint32)\n                    else:\n                        df[col] = df[col].astype(np.uint64)\n                else:\n                    if mn > np.iinfo(np.int8).min and mx < np.iinfo(np.int8).max:\n                        df[col] = df[col].astype(np.int8)\n                    elif mn > np.iinfo(np.int16).min and mx < np.iinfo(np.int16).max:\n                        df[col] = df[col].astype(np.int16)\n                    elif mn > np.iinfo(np.int32).min and mx < np.iinfo(np.int32).max:\n                        df[col] = df[col].astype(np.int32)\n                    elif mn > np.iinfo(np.int64).min and mx < np.iinfo(np.int64).max:\n                        df[col] = df[col].astype(np.int64)    \n            # Make float datatypes 32 bit\n            else:\n                df[col] = df[col].astype(np.float32)\n            \n            # Print new column type\n            #print(\"dtype after: \",df[col].dtype)\n#             print(\"******************************\")\n    # Print final result\n    print(\"___MEMORY USAGE AFTER COMPLETION:___\")\n    mem_usg = df.memory_usage().sum() / 1024**2 \n    print(\"Memory usage is: \",mem_usg,\" MB\")\n    print(\"This is \",100*mem_usg/start_mem_usg,\"% of the initial size\")\n    return df, NAlist","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, _ = reduce_mem_usage(train)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Transforming timestamp into Month, DayofMonth, Weekday, Time"},{"metadata":{"trusted":true},"cell_type":"code","source":"for df in [train]:#, test]:\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n    df['Month'] = df['timestamp'].dt.month.astype(\"uint8\")\n    df['DayOfMonth'] = df['timestamp'].dt.day.astype(\"uint8\")\n    df['Weekend'] = df['timestamp'].dt.weekday.astype(\"uint8\")\n    df['Hour'] = df['timestamp'].dt.hour.astype(\"uint8\")\n    df.drop('timestamp', axis=1, inplace=True)\n    gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"categoricals = [\"building_id\", \"primary_use\", \"Hour\", \"Weekend\", \"Month\", \"meter\", \"wind_direction\"]\nnumericals = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\", \"dew_temperature\", \"floor_count\", \"precip_depth_1_hr\", \"sea_level_pressure\", \"wind_speed\"]\n\nfeat_cols = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = train['meter_reading']\n\ntrain.drop(['site_id', 'meter_reading'], axis=1, inplace=True)\ntrain = train[feat_cols]\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inspiration from the Half and Half notebook Using a diferent algorithm: xgboost, which is also a gradient boosting algorithm. The idea is to train two models, both on half the training datasets and validating on the other half. Predictions works by averaging their respecctive predictions after exponenting the logarithmic predictions, hoping that this would help with not overfitting the models."},{"metadata":{"trusted":true},"cell_type":"code","source":"x_half_1 = train[:int(train.shape[0] / 2)]\nx_half_2 = train[int(train.shape[0] / 2):]\n\ny_half_1 = y[:int(train.shape[0] / 2)]\ny_half_2 = y[int(train.shape[0] / 2):]\n\nxgb_half_1 = xgb.DMatrix(x_half_1, y_half_1)\nxgb_half_2 = xgb.DMatrix(x_half_2, y_half_2)\n\npars = {\n    'colsample_bytree': 0.8,                 \n    'learning_rate': 0.3,\n    'max_depth': 5,\n    'subsample': 0.8,\n    'objective': 'reg:squarederror',\n    #'eval_metric': 'rmsle',\n}\n\nmodel_half_1 = xgb.train(pars,\n                  xgb_half_1,\n                  num_boost_round=500,\n                  evals=[(xgb_half_1, 'train'), (xgb_half_2, 'val')],\n                  verbose_eval=5,\n                  early_stopping_rounds=5\n                 )\nmodel_half_2 = xgb.train(pars,\n                  xgb_half_2,\n                  num_boost_round=100,\n                  evals=[(xgb_half_2, 'train'), (xgb_half_1, 'val')],\n                  verbose_eval=5,\n                  early_stopping_rounds=5\n                 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# in case of memory issuese we have something to refer back to\n# save model to file\n#pickle.dump(model_half_1, open(\"mh1.pickle.dat\", \"wb\"))\n# save model to file\n#pickle.dump(model_half_2, open(\"mh2.pickle.dat\", \"wb\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train, x_half_1, x_half_2, y_half_1, y_half_2, xgb_half_1, xgb_half_2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Only for test data, so we wouldn't fill RAM in the beginning\n\nweather_test = pd.read_csv(\"../input/ashrae-energy-prediction/weather_test.csv\")#, parse_dates=['timestamp'], dtype=weather_dtype)\ntest = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")#, parse_dates=['timestamp'], usecols=['building_id','meter','timestamp'], dtype=train_dtype)\n\ntest  = pd.merge(test,metadata,on='building_id',how='left')\nprint (\"Testing Data+Metadata Shape {}\".format(test.shape))\ndel metadata\ngc.collect()\n\ntest  = pd.merge(test,weather_test,on=['site_id','timestamp'],how='left')\nprint (\"Testing Data+Metadata+Weather Shape {}\".format(test.shape))\ndel weather_test\ngc.collect()\n\ntest[\"primary_use\"] = le.transform(test[\"primary_use\"])\n\ntest, _ = reduce_mem_usage(test)\ngc.collect()\n\nfor df in [test]:\n    df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n    df['Month'] = df['timestamp'].dt.month.astype(\"uint8\")\n    df['DayOfMonth'] = df['timestamp'].dt.day.astype(\"uint8\")\n    df['Weekend'] = df['timestamp'].dt.weekday.astype(\"uint8\")\n    df['Hour'] = df['timestamp'].dt.hour.astype(\"uint8\")\n    df.drop('timestamp', axis=1, inplace=True)\n    gc.collect()\n    \ntest = test[feat_cols]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = []\nstep_size = 50000\nfor i in range(int(np.ceil(test.shape[0]/step_size))):\n    h1 = np.expm1(model_half_1.predict(xgb.DMatrix(test.iloc[i*step_size:(i+1)*step_size])))\n    h2 = np.expm1(model_half_2.predict(xgb.DMatrix(test.iloc[i*step_size:(i+1)*step_size])))\n    result.append((h1+h2)/2)\n\ndel test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result = np.concatenate(result)\nsubmission = pd.read_csv(\"sample_submission.csv\")\nsubmission[\"meter_reading\"] = result\nsubmission.to_csv(\"submission.csv\", index = False)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The following notebooks were read and used as inspiration:\n\nhttps://www.kaggle.com/caesarlupum/ashrae-start-here-a-gentle-introduction\n\nhttps://www.kaggle.com/rohanrao/ashrae-half-and-half\n\nhttps://www.kaggle.com/kailex/ac-dc\n\nhttps://www.kaggle.com/ihelon/simple-xgboost\n\nhttps://www.kaggle.com/morituri/lgbm-baseline\n\nhttps://www.kaggle.com/arjanso/reducing-dataframe-memory-size-by-65\n"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}