{"cells":[{"metadata":{"_uuid":"1ca3b5c7-9a6a-4c98-b9da-f8b9819d5dc3","_cell_guid":"813f3d47-3aef-4942-8f80-9eafbb1f3b1d","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport multiprocessing\nimport warnings\nimport os\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport lightgbm as lgb\nimport gc\nfrom time import time\nimport datetime\nfrom tqdm import tqdm_notebook\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.model_selection import KFold, TimeSeriesSplit\nfrom sklearn.metrics import roc_auc_score\nwarnings.simplefilter('ignore')\nsns.set()\n%matplotlib inline\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b942253f-41b3-4f4c-a953-0fc8ae11a15d","_cell_guid":"2b48a6c2-3748-4096-b3cb-6ad779f352e8","trusted":true},"cell_type":"code","source":"from pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c33b2d7-b28d-48e3-b026-b9741f7dbdcd","_cell_guid":"b42b456a-f18e-4744-bb47-2b19ad12ee1a","trusted":true},"cell_type":"code","source":"# read the data\n#building is common on both train and test\n# we have separate weather for both train and test\ntrain_df=pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/train.csv\")\nbuilding_df=pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/building_metadata.csv\")\nweather_train_df=pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/weather_train.csv\")\nweather_test_df=pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/weather_test.csv\")\ntest_df=pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/test.csv\")\nprint(\"Data readed successfully\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6f29d113-8f9e-4374-9177-84892d174a59","_cell_guid":"ecbb5707-7518-4921-8cf3-f520bf6f97f7","trusted":true},"cell_type":"code","source":"# reduce the memory to join the\ntrain_df=reduce_mem_usage(train_df,use_float16=True)\nbuilding_df=reduce_mem_usage(building_df,use_float16=True)\nweather_train_df=reduce_mem_usage(weather_train_df,use_float16=True)\nweather_test_df=reduce_mem_usage(weather_test_df,use_float16=True)\ntest_df=reduce_mem_usage(test_df,use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"74096d6d-8128-4e32-b038-d0e11d2fa413","_cell_guid":"e4d60a3c-f899-4a21-bb81-1ea92a5a716a","trusted":true},"cell_type":"code","source":"# we need to combine the different csv into single one for analysis\n# for train  we need to combine the building metadata with building_id and site_id for the weather files.\n# same as test\ntrain_df=train_df.merge(building_df,on='building_id',how='left')\ntrain_df=train_df.merge(weather_train_df,on=['site_id', 'timestamp'], how='left')\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3660dc6b-a405-4480-b0a4-b8bbd233939c","_cell_guid":"3d2ca71a-1fc6-4382-8b93-3b1571e77c5b","trusted":true},"cell_type":"code","source":"# same for the test \ntest_df=test_df.merge(building_df,on='building_id',how='left')\ntest_df=test_df.merge(weather_test_df,on=['site_id', 'timestamp'], how='left')\ntest_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=reduce_mem_usage(train_df,use_float16=True)\ntest_df=reduce_mem_usage(test_df,use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"534d2939-62dc-4fe0-8aca-2b9092a214e8","_cell_guid":"853b7619-7891-49f6-a0b9-a7be7d164b03","trusted":true},"cell_type":"code","source":"# delete the variables for memory problem\ngc.collect()\ndel building_df,weather_train_df,weather_test_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54e5a54f-012c-4a61-9a07-8ee82429d033","_cell_guid":"fc8da910-d853-48f2-90f1-5d134d8817b1","trusted":true},"cell_type":"code","source":"# we have combine the data successfully how we can start to analyze the data\n# we can start\nprint(train_df.columns)\nprint(\"The total number of columns\",len(train_df.columns))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cecd09eb-8b4d-4339-bf67-97a0fdcd9d18","_cell_guid":"02eb6013-7936-4b36-b435-9c4a1a85f7e8","trusted":true},"cell_type":"code","source":"train_df.timestamp = pd.to_datetime(train_df.timestamp)\ntest_df.timestamp=pd.to_datetime(test_df.timestamp)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a64f909-b18c-4039-8a47-3346f767d328","_cell_guid":"ee437c37-a385-42e4-a0a9-de6a7dbec691","trusted":true},"cell_type":"code","source":"#its time series data we need to take care of \ntrain_df[\"hour\"] = train_df[\"timestamp\"].dt.hour\ntrain_df[\"day\"] = train_df[\"timestamp\"].dt.day\ntrain_df[\"weekend\"] = train_df[\"timestamp\"].dt.weekday\ntrain_df[\"month\"] = train_df[\"timestamp\"].dt.month\ntrain_df[\"dayofweek\"] = train_df[\"timestamp\"].dt.dayofweek\ntrain_df[\"Year\"] = train_df[\"timestamp\"].dt.year","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test data\n#its time series data we need to take care of \ntest_df[\"hour\"] = test_df[\"timestamp\"].dt.hour\ntest_df[\"day\"] = test_df[\"timestamp\"].dt.day\ntest_df[\"weekend\"] = test_df[\"timestamp\"].dt.weekday\ntest_df[\"month\"] = test_df[\"timestamp\"].dt.month\ntest_df[\"dayofweek\"] = test_df[\"timestamp\"].dt.dayofweek\ntest_df[\"Year\"] = test_df[\"timestamp\"].dt.year","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for cols in train_df.columns:\n    if cols not in test_df.columns:\n        print(cols)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# feature engineering\n# need for the columns \n\"\"\"[ 'meter', 'site_id',\n       'primary_use', 'square_feet', 'year_built', 'floor_count',\n       'air_temperature', 'cloud_coverage', 'dew_temperature',\n       'precip_depth_1_hr', 'sea_level_pressure', 'wind_direction',\n       'wind_speed']\"\"\"\n#air_temperature,dew_temperature,precip_depth_1_hr,sea_level_pressure,wind_direction,wind_speed\ntransform_col=['air_temperature','dew_temperature','precip_depth_1_hr','sea_level_pressure','wind_direction','wind_speed']\n\n#train_df['air_temperature_mean']=train_df.groupby(['site_id','month','dayofweek'])['air_temperature'].transform('mean')\nfor col in transform_col:\n    train_df[col+\"_mean\"]=train_df.groupby(['site_id','month','dayofweek'])[col].transform('mean')\n    train_df[col+\"_std\"]=train_df.groupby(['site_id','month','dayofweek'])[col].transform('std')\n    test_df[col+\"_mean\"]=test_df.groupby(['site_id','month','dayofweek'])[col].transform('mean')\n    test_df[col+\"_std\"]=test_df.groupby(['site_id','month','dayofweek'])[col].transform('std')\n    del train_df[col],test_df[col]\nfor col in train_df.columns:\n    if col not in test_df.columns:\n        print(\"Col:\",col)\n\n\n#train_df.groupby(['site_id','month','dayofweek'])['air_temperature'].transform('std')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df=reduce_mem_usage(train_df,use_float16=True)\ntest_df=reduce_mem_usage(test_df,use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()\ncategorical_cols = ['site_id','meter','primary_use','hour','day','weekend','Year','month','dayofweek']\nfor c in categorical_cols:\n    train_df[c]=pd.Categorical(train_df[c])\n    test_df[c]=pd.Categorical(test_df[c])\n# now we can bulid the model on the top of this dataframe","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df['building_id'],test_df['building_id']\ndel test_df['row_id']\nfor col in test_df.columns:\n    if col not in train_df.columns:\n        print(\"Missing:\",col)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y=train_df['meter_reading']\ndel train_df['meter_reading']\ngc.collect()\nlist1=train_df.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_df['timestamp'],test_df['timestamp']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nx_train,x_test,y_train,y_test = train_test_split(train_df,y,test_size=0.25,random_state=42)\nprint (x_train.shape)\nprint (y_train.shape)\nprint (x_test.shape)\nprint (y_test.shape)\n\nlgb_train = lgb.Dataset(x_train, y_train,categorical_feature=categorical_cols)\nlgb_test = lgb.Dataset(x_test, y_test,categorical_feature=categorical_cols)\ndel x_train, x_test , y_train, y_test\n\nparams = {'feature_fraction': 0.75,\n          'bagging_fraction': 0.75,\n          'objective': 'regression',\n          'max_depth': -1,\n          'learning_rate': 0.15,\n        #  \"boosting_type\": \"gbdt\",\n          \"bagging_seed\": 11,\n          \"metric\": 'rmse',\n          \"verbosity\": -1,\n          'reg_alpha': 0.5,\n          'reg_lambda': 0.5,\n          'random_state': 47\n         }\n\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reg = lgb.train(params, lgb_train, num_boost_round=3000, valid_sets=[lgb_test], early_stopping_rounds=100, verbose_eval = 100)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del lgb_train,lgb_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(reg.feature_importance())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del list1[1]\nli=list(list1)\ndel li[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nser = pd.DataFrame(reg.feature_importance(),li,columns=['Importance']).sort_values(by='Importance')\nser['Importance'].plot(kind='bar',figsize=(10,6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\npredictions = []\n#https://www.kaggle.com/kulkarnivishwanath/ashrae-great-energy-predictor-iii-eda-model\nstep = 50000\nfor i in range(0, len(test_df), step):\n    print(str(i)+\"Steps-----------------\")\n    predictions.extend(np.expm1(reg.predict(test_df.iloc[i: min(i+step, len(test_df)), :], num_iteration=reg.best_iteration)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions[1:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df=pd.read_csv(\"/kaggle/input/ashrae-energy-prediction/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df['meter_reading']=predictions\nsample_df['meter_reading'].clip(lower=0,upper=None,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_df.to_csv(\"lgbm_model_1.csv\",index=None)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}