{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Importing Libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# Suppress warnings \nimport warnings\nwarnings.filterwarnings('ignore')\nimport gc\n\n\n# matplotlib and seaborn for plotting\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nimport matplotlib.patches as patches\nfrom scipy import stats\nfrom scipy.stats import skew\n\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nimport plotly.express as px\npd.set_option('max_columns', 100)\n\npy.init_notebook_mode(connected=True)\nfrom plotly.offline import init_notebook_mode, iplot\ninit_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nimport os,random, math, psutil, pickle\nfrom sklearn.preprocessing import LabelEncoder\n\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb\nfrom sklearn.model_selection import KFold, StratifiedKFold\nfrom tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading and understanding our data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=pd.read_csv(\"../input/ashrae-energy-prediction/train.csv\")\ntest_df = pd.read_csv(\"../input/ashrae-energy-prediction/test.csv\")\nweather_train_df = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv')\nweather_test_df = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv')\nbuilding_meta_df = pd.read_csv('../input/ashrae-energy-prediction/building_metadata.csv')\nsample_submission = pd.read_csv('../input/ashrae-energy-prediction/sample_submission.csv')\n\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Size of train_df data', train_df.shape)\nprint('Size of weather_train_df data', weather_train_df.shape)\nprint('Size of weather_test_df data', weather_test_df.shape)\nprint('Size of building_meta_df data', building_meta_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reduce The Memory For Faster Processing"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Function to reduce the DF size\ndef reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = reduce_mem_usage(train_df)\ntest_df = reduce_mem_usage(test_df)\n\nweather_train_df = reduce_mem_usage(weather_train_df)\nweather_test_df = reduce_mem_usage(weather_test_df)\nbuilding_meta_df = reduce_mem_usage(building_meta_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"building_meta_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\nFeatures that are likely predictive:\n\nBuildings\n\n    primary_use\n    square_feet\n    year_built\n    floor_count (may be too sparse to use)\n\nWeather\n\n    time of day\n    holiday\n    weekend\n    cloud_coverage + lags\n    dew_temperature + lags\n    precip_depth + lags\n    sea_level_pressure + lags\n    wind_direction + lags\n    wind_speed + lags\n\nTrain\n\n    max, mean, min, std of the specific building historically\n    number of meters\n    number of buildings at a siteid\n\n"},{"metadata":{},"cell_type":"markdown","source":"**Converting \"timestamps\" To datetime And String To Category Datatype**"},{"metadata":{},"cell_type":"markdown","source":"*Categoricals are a pandas data type corresponding to categorical variables in statistics. A categorical variable takes on a limited, and usually fixed, number of possible values*"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['timestamp'] = pd.to_datetime(train_df['timestamp'])\ntest_df['timestamp'] = pd.to_datetime(test_df['timestamp'])\n\nweather_train_df['timestamp'] = pd.to_datetime(weather_train_df['timestamp'])\nweather_test_df['timestamp'] = pd.to_datetime(weather_test_df['timestamp'])\n\n#building_meta_df['primary_use'] = building_meta_df['primary_use'].astype('category')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[\"hour\"] = train_df[\"timestamp\"].dt.hour\ntrain_df[\"day\"] = train_df[\"timestamp\"].dt.day\ntrain_df[\"weekend\"] = train_df[\"timestamp\"].dt.weekday\ntrain_df[\"month\"] = train_df[\"timestamp\"].dt.month\n\ntest_df[\"hour\"] = test_df[\"timestamp\"].dt.hour\ntest_df[\"day\"] = test_df[\"timestamp\"].dt.day\ntest_df[\"weekend\"] = test_df[\"timestamp\"].dt.weekday\ntest_df[\"month\"] = test_df[\"timestamp\"].dt.month","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merging the features of building_meta_df with train_df and test_df"},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['building_id']]\ntemp_df = temp_df.merge(building_meta_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ntemp_df = test_df[['building_id']]\ntemp_df = temp_df.merge(building_meta_df, on=['building_id'], how='left')\ndel temp_df['building_id']\ntest_df = pd.concat([test_df, temp_df], axis=1)\n\ndel temp_df, building_meta_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Merging the features of weather_df with train_df and test_df"},{"metadata":{"trusted":true},"cell_type":"code","source":"temp_df = train_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(weather_train_df, on=['site_id','timestamp'], how='left')\n\ndel temp_df['site_id'], temp_df['timestamp']\ntrain_df = pd.concat([train_df, temp_df], axis=1)\n\ntemp_df = test_df[['site_id','timestamp']]\ntemp_df = temp_df.merge(weather_test_df, on=['site_id','timestamp'], how='left')\n\ndel temp_df['site_id'], temp_df['timestamp']\ntest_df = pd.concat([test_df, temp_df], axis=1)\n\ndel temp_df, weather_train_df, weather_test_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Storing train_df and test_df using to_pickle**\n\n*to_pickle(): A good solution for keeping that dataframe constantly available in between runs so I don't have to spend all that time waiting for the script to run*\n\n*gc.collect() : It performs a blocking garbage collection of all generations. All objects, regardless of how long they have been in memory, are considered for collection; however, objects that are referenced in managed code are not collected. Use this method to force the system to try to reclaim the maximum amount of available memory.*"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.to_pickle('train_df.pkl')\ntest_df.to_pickle('test_df.pkl')\n   \ndel train_df, test_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_pickle('train_df.pkl')\ntest_df = pd.read_pickle('test_df.pkl')\n\ntrain_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Featuring Engineering"},{"metadata":{},"cell_type":"markdown","source":"**Encoding**"},{"metadata":{"trusted":true},"cell_type":"code","source":"le = LabelEncoder()\n\ntrain_df['primary_use'] = le.fit_transform(train_df['primary_use']).astype(np.int8)\n\ntest_df['primary_use'] = le.fit_transform(test_df['primary_use']).astype(np.int8)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Calculating The Age**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_df['age'] = train_df['year_built'].max() - train_df['year_built'] + 1\n#test_df['age'] = test_df['year_built'].max() - test_df['year_built'] + 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Find the optimal feature subset using an evaluation measure. The choice of evaluation metric distinguish the three main strategies of feature selection algorithms: the wrapper strategy, the filter strategy, and the embedded strategy.**\n\nFilter methods:\n\n    information gain\n    chi-square test\n    correlation coefficient\n    variance threshold\n    \n    \nWrapper methods:\n\n    recursive feature elimination\n    sequential feature selection algorithms\n    \n    \nEmbedded methods:\n\n    L1 (LASSO) regularization\n    decision tree"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Dropping Timestamps**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(\"timestamp\", axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = test_df.drop(\"timestamp\", axis = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dealing With Cat,Num Features And Dropping Unnecessary Features"},{"metadata":{"trusted":true},"cell_type":"code","source":"categoricals = [\"building_id\", \"primary_use\", \"hour\", \"day\", \"weekend\", \"month\", \"meter\"]\n\nnumericals = [\"square_feet\", \"year_built\", \"air_temperature\", \"cloud_coverage\",\"dew_temperature\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"drop_cols = [\"precip_depth_1_hr\", \"sea_level_pressure\", \"wind_direction\", \"wind_speed\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df[categoricals + numericals]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[categoricals + numericals]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feat_cols = categoricals + numericals","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#plotting 100 highest consuming buildings\n#import matplotlib.pyplot as plt\n#top_buildings = train_df.groupby(\"building_id\")[\"meter_reading\"].mean().sort_values(ascending = False).iloc[:100]\n\n#for value in top_buildings.index:\n#    train_df[train_df[\"building_id\"] == value][\"meter_reading\"].rolling(window = 24).mean().plot()\n#    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\ntrain_df.hist(figsize=(10,10),color=\"green\",bins=20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"*When your input value is so small, using np.log1p to calculate, you will get the more accutrate result than np.log according to the interpretation from*"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.log1p(train_df[\"meter_reading\"])\ndel train_df[\"meter_reading\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = train_df.drop(drop_cols + [\"site_id\", \"floor_count\"], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = test_df.drop(drop_cols + [\"site_id\", \"floor_count\",\"row_id\"], axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modelling"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import KFold\nfrom sklearn.metrics import mean_squared_error\nimport lightgbm as lgb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Cross-validation is a resampling procedure used to evaluate machine learning models on a limited data sample.**\n\nThe general procedure is as follows:\n\n    Shuffle the dataset randomly.\n    \n    Split the dataset into k groups\n    \n    For each unique group:\n    \n        Take the group as a hold out or test data set\n        Take the remaining groups as a training data set\n        Fit a model on the training set and evaluate it on the test set\n        Retain the evaluation score and discard the model\n        \n    Summarize the skill of the model using the sample of model evaluation scores\n    \n"},{"metadata":{},"cell_type":"markdown","source":"**You can see that I'm not actually doing Kfold. I am skipping the first 4 folds and then just doing the last one so I am actually training on the first 80% of data and validating on the last 20%. I intended to do kfold and then realized it was too compute heavy and may possibly cause leakage.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_folds = 5\nkf = KFold(n_splits = num_folds, shuffle = False, random_state = 42)\nerror = 0\nmodels = []\nfor i, (train_index, val_index) in enumerate(kf.split(train_df)):\n    if i + 1 < num_folds:\n        continue\n    print(train_index.max(), val_index.min())\n    \n    #Splitting The DF Into Train Test Split\n    train_X = train_df[feat_cols].iloc[train_index]\n    val_X = train_df[feat_cols].iloc[val_index]\n    train_y = target.iloc[train_index]\n    val_y = target.iloc[val_index]\n    \n    #Training The Model\n    lgb_train = lgb.Dataset(train_X, train_y > 0)\n    lgb_eval = lgb.Dataset(val_X, val_y > 0)\n    params = {\n            'boosting_type': 'gbdt',\n            'objective': 'binary',\n            'metric': {'binary_logloss'},\n            'learning_rate': 0.1,\n            'feature_fraction': 0.8,\n            'bagging_fraction': 0.8,\n            'bagging_freq' : 5\n            }\n    gbm_class = lgb.train(params,\n                lgb_train,\n                num_boost_round=2000,\n                valid_sets=(lgb_train, lgb_eval),\n               early_stopping_rounds=20,\n               verbose_eval = 20)\n    \n    #we did this so that we only consider positive vals and no dta leakage would be there\n    lgb_train = lgb.Dataset(train_X[train_y > 0], train_y[train_y > 0]) \n    lgb_eval = lgb.Dataset(val_X[val_y > 0] , val_y[val_y > 0])\n    params = {\n            'boosting_type': 'gbdt',\n            'objective': 'regression',\n            'metric': {'rmse'},\n            'learning_rate': 0.5,\n            'feature_fraction': 0.8,\n            'bagging_fraction': 0.8,\n            'bagging_freq' : 5\n            }\n    gbm_regress = lgb.train(params,\n                lgb_train,\n                num_boost_round=2000,\n                valid_sets=(lgb_train, lgb_eval),\n               early_stopping_rounds=20,\n               verbose_eval = 20)\n#     models.append(gbm)\n\n    y_pred = (gbm_class.predict(val_X, num_iteration=gbm_class.best_iteration) > .5) *\\\n    (gbm_regress.predict(val_X, num_iteration=gbm_regress.best_iteration))\n    error += np.sqrt(mean_squared_error(y_pred, (val_y)))/num_folds\n    print(np.sqrt(mean_squared_error(y_pred, (val_y))))\n    break\nprint(error)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted(zip(gbm_regress.feature_importance(), gbm_regress.feature_name()),reverse = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Modifying For Submissions"},{"metadata":{},"cell_type":"markdown","source":"*tqdm: Is Used For Displaying Horizontal Animated Bar*\n\n*\\ : Is For Regular Expression"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\ni=0\nres=[]\nstep_size = 50000\nfor j in tqdm(range(int(np.ceil(test_df.shape[0]/50000)))):\n    \n    res.append(np.expm1\n               (\n                   (gbm_class.predict\n                    (test_df.iloc[i:i+step_size], num_iteration=gbm_class.best_iteration) > .5) *\\\n                       (gbm_regress.predict(test_df.iloc[i:i+step_size], num_iteration=gbm_regress.best_iteration))\n               )\n              )\n    i+=step_size","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res = np.concatenate(res)\npd.DataFrame(res).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(res)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"../input/ashrae-energy-prediction/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[\"meter_reading\"] = res","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.remove(\"./train_df.pkl\")\nos.remove(\"./test_df.pkl\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv(\"submission.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}