{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport os\nimport random\nimport pickle\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport lightgbm as lgb\n\nfrom matplotlib import pyplot as plt\nfrom sklearn.preprocessing import LabelEncoder\n\npath_data = \"/kaggle/input/ashrae-energy-prediction/\"\npath_train = path_data + \"train.csv\"\npath_test = path_data + \"test.csv\"\npath_building = path_data + \"building_metadata.csv\"\npath_weather_train = path_data + \"weather_train.csv\"\npath_weather_test = path_data + \"weather_test.csv\"\n\nplt.style.use(\"seaborn\")\nsns.set(font_scale=1)\n\nmyfavouritenumber = 0\nseed = myfavouritenumber\nrandom.seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Reading train data\nReading train data along with building and weather metadata."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"%time df_train = pd.read_csv(path_train)\n%time df_test = pd.read_csv(path_test)\n%time building = pd.read_csv(path_building)\n%time weather_train = pd.read_csv(path_weather_train)\n%time weather_test = pd.read_csv(path_weather_test)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"## Memory optimization\n\n# Original code from https://www.kaggle.com/gemartin/load-data-reduce-memory-usage by @gemartin\n# Modified to support timestamp type, categorical type\n# Modified to add option to use float16\n\nfrom pandas.api.types import is_datetime64_any_dtype as is_datetime\nfrom pandas.api.types import is_categorical_dtype\n\ndef reduce_mem_usage(df, use_float16=False):\n    \"\"\"\n    Iterate through all the columns of a dataframe and modify the data type to reduce memory usage.        \n    \"\"\"\n    \n    start_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage of dataframe is {:.2f} MB\".format(start_mem))\n    \n    for col in df.columns:\n        if is_datetime(df[col]) or is_categorical_dtype(df[col]):\n            continue\n        col_type = df[col].dtype\n        \n        if col_type != object:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == \"int\":\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if use_float16 and c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)\n        else:\n            df[col] = df[col].astype(\"category\")\n\n    end_mem = df.memory_usage().sum() / 1024**2\n    print(\"Memory usage after optimization is: {:.2f} MB\".format(end_mem))\n    print(\"Decreased by {:.1f}%\".format(100 * (start_mem - end_mem) / start_mem))\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = reduce_mem_usage(df_train, use_float16=True)\ndf_test = reduce_mem_usage(df_test,use_float16=True)\nbuilding = reduce_mem_usage(building, use_float16=True)\nweather_train = reduce_mem_usage(weather_train, use_float16=True)\nweather_test = reduce_mem_usage(weather_test, use_float16=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train.merge(building, on=\"building_id\", how=\"left\")\ndf_train = df_train.merge(weather_train, on=[\"site_id\", \"timestamp\"], how=\"left\")\ndf_test = df_test.merge(building, on=\"building_id\", how=\"left\")\ndf_test = df_test.merge(weather_test, on=[\"site_id\", \"timestamp\"], how=\"left\")\ndel weather_train,weather_test;gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.shape,df_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train = df_train.drop(\"building_id\",axis=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### explore about \"meter_reading\" "},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15,5))\ndf_train['meter_reading'].plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['meter_reading'].plot(kind='hist',\n                              bins=15,\n                              figsize=(15,5),\n                              title='distribution of \"meter_reading\"')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* the number of 0 or about 0 is too much.\n* when meter_reading after a certain time, the use of energy is reduced to a low level.\n## other variables"},{"metadata":{"trusted":true},"cell_type":"code","source":"# sort data by \"timestamp\"\ndf_train = df_train.sort_values(by=\"timestamp\" , ascending=True) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### correlation of other vaiables and meter_reading"},{"metadata":{"trusted":true},"cell_type":"code","source":"correlation = df_train.corr()\ncorrelation['meter_reading'].sort_values()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"correlation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (15,10))\nsns.heatmap(correlation,cmap=plt.cm.RdYlBu_r,vmin=-0.25,\n            annot=True,vmax=0.6)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### building_id and meter"},{"metadata":{"trusted":true},"cell_type":"code","source":"# building_id\nprint('the number of building_id:{}'.format(df_train.building_id.nunique()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"meter1 = df_train['meter_reading'].loc[df_train.meter==0].groupby(df_train.building_id).mean()\nmeter2 = df_train['meter_reading'].loc[df_train.meter==1].groupby(df_train.building_id).mean()\nmeter3 = df_train['meter_reading'].loc[df_train.meter==2].groupby(df_train.building_id).mean()\nmeter4 = df_train['meter_reading'].loc[df_train.meter==3].groupby(df_train.building_id).mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,5))\nmeter1.plot(kind='line',logy=True,color='green')\nmeter2.plot(kind='line',color='blue')\nmeter3.plot(kind='line',color='yellow')\nmeter4.plot(kind='line',color='red')\nplt.legend(['electricity','chilledwater','steam','hotwater'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['meter_reading'].groupby(df_train.meter).mean()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**the steam might be use much more power!!!**"},{"metadata":{"trusted":true},"cell_type":"code","source":"len(df_train.building_id)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"build_meter = df_train.meter.groupby(df_train.building_id).nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.meter_reading.loc[df_train.building_id==5].sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = [0,0,0,0]\nnum_x = [0,0,0,0]\nfor i in range(len(build_meter)):\n    heat = df_train.meter_reading.loc[df_train.building_id==i].sum()\n    x[build_meter[i]-1] = x[build_meter[i]-1] + heat\n    num_x[build_meter[i]-1] = num_x[build_meter[i]-1]+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.DataFrame(x).plot(kind='bar',title='influence(max) of the numeber of meter categories',xticks=[1,2,3,4])\nfor i in range(4):\n    x[i] = x[i]/num_x[i]\npd.DataFrame(x).plot(kind='bar',title='influence(mean by building num) of the numeber of meter categories',xticks=[1,2,3,4])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**if some buildings have 4 kinds of meters, it must cost much more energy. Next is two kinds of meters**\n**what is interesting is the three kinds of meters, which doesn't cost more energy than two**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_col(column):\n    plt.subplots(figsize=(6,6))\n    sns.distplot(df_train[column],color='green').set_title(column)\n    sns.distplot(df_test[column],color='yellow').set_title(column)\n    plt.legend(['train','test'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# meter\nplt.subplot(2,2,1)\ndf_train['meter'].plot(kind='hist',bins=4,figsize=(16,5),xticks=[0,1,2,3])\nplt.title(\"the number of meter(0: electricity, 1: chilledwater, 2: steam, 3: hotwater)\")\nave = []\nfor i in range(4):\n    ave.append(df_train.loc[(df_train.meter==i)].meter_reading.mean())\nplt.subplot(2,2,2,title='the mean of meter_reading')\nplt.plot(ave)\nplt.subplot(2,2,3,title='the number of meter_reading=0')\nave = []\nfor i in range(4):\n    ave.append(df_train.meter_reading.loc[(df_train.meter==i) & (df_train.meter_reading==0)].count())\nplt.plot(ave)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"2:steam are likely to have higher meter_reading"},{"metadata":{"trusted":true},"cell_type":"code","source":"# site_id\nplt.subplot(1,2,1)\ndf_train['site_id'].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(14):\n    ave.append(df_train.loc[(df_train.site_id==i)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(ave)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\"site_id\"=14 are likely to have higher meter_reading.\n\nHowever, \"site_id\" is set for foreign key at first so I am not sure if this is a good feature."},{"metadata":{"trusted":true},"cell_type":"code","source":"# square_feet - Gross floor area of the building\nplt.subplot(1,2,1)\ndf_train['square_feet'].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(8):\n    ave.append(df_train.loc[(df_train.square_feet/100000<i+1) & (df_train.square_feet/100000>=i)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(ave)\nplot_col('square_feet')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"When \"square_feet\" is between 200000 and 300000, \"meter_reading\" is higher."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.subplot(1,2,1)\ndf_train['year_built'].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(10):\n    ave.append(df_train.loc[(df_train.year_built>1900+i*10) & (df_train.year_built<=1910+i*10)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(ave)\nprint(\"number of nan in 'year_build':{},pencent:{}%\".format(df_train.meter_reading.loc[df_train.year_built.isnull()].count(),100*df_train.meter_reading.loc[df_train.year_built.isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when 'year_build' is nan:{}\".format(df_train.meter_reading.loc[df_train.year_built.isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* when \"year_built\" is big, the mean of \"meter_reading\" is higher\n* There are 60% data without a \"year_built\" and the \"meter_reading\" is much higher when \"year_built\" is nan!.\n\nAdd a new column \"year_built_ifnan\"\n\n\"year_built\"------nan:0\n\nmaybe we should delete \"year_built\""},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['year_built_ifnan'] = df_train.year_built.isnull().astype('int')\ndf_train['year_built'] = df_train['year_built'].fillna(2018)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# floor_count\nplt.subplot(1,2,1)\ndf_train['floor_count'].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(10):\n    ave.append(df_train.loc[(df_train.floor_count>i*2.5) & (df_train.floor_count<2.5+i*2.5)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(ave)\nprint(\"number of nan in 'floor_count':{},pencent:{}%\".format(df_train.meter_reading.loc[df_train.floor_count.isnull()].count(),100*df_train.meter_reading.loc[df_train.floor_count.isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when 'floor_count' is nan:{}\".format(df_train.meter_reading.loc[df_train.floor_count.isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* There are 82% data without a \"floor_count\" and the \"meter_reading\" is much higher when \"year_built\" is nan!.\n\nAdd a new column \"floor_count_ifnan\"\n\ndelete \"floor_count\""},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['floor_count_ifnan'] = df_train.floor_count.isnull().astype('int')\ndf_train = df_train.drop('floor_count',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# air_temperature\nfeature = \"air_temperature\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(10):\n    ave.append(df_train.loc[(df_train[feature]>-30+i*10) & (df_train[feature]<-20+i*10)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(np.arange(-25,75,10),ave)\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* percent of nan is only 0.47%\n* the colder, the higher\n\nfillna(35) "},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"air_temperature\"] = df_train[\"air_temperature\"].fillna(35)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cloud_coverage\nfeature = \"cloud_coverage\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nplt.title('Portion of the sky covered in clouds, in oktas')\nave = []\nfor i in [0,2,4,6,8]:\n    ave.append(df_train.loc[(df_train[feature]==i)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(np.arange(0,10,2),ave)\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* the higher portion is, the higher the meter_reading is\n* the nan portion is 43%\n\nadd column \"cloud_coverage_ifnan\"\nfillna(5)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['cloud_coverage_ifnan'] = df_train.cloud_coverage.isnull().astype(\"int\")\ndf_train['cloud_coverage'] = df_train.cloud_coverage.fillna(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['primary_use_encoded'] = LabelEncoder().fit_transform(df_train.primary_use).astype(\"int\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# primary_use\nfeature = \"primary_use_encoded\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nplt.title('Indicator of the primary category of activities for the building')\nave = []\nfor i in range(16):\n    ave.append(df_train.loc[(df_train[feature]==i)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(np.arange(0,16,1),ave)\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I have a feeling that I should use the count() of each use as encoding method."},{"metadata":{"trusted":true},"cell_type":"code","source":"primary_use_num = {}\nfor i in range(16):\n    primary_use_num[i] = df_train.meter_reading.loc[df_train.primary_use_encoded==i].count()\ndf_train['primary_use_encoded'] = df_train.primary_use_encoded.map(primary_use_num)\ndf_train = df_train.drop('primary_use',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"primary_use_num","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# dew_temperature\nfeature = \"dew_temperature\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(10):\n    ave.append(df_train.loc[(df_train[feature]>-29+i*6) & (df_train[feature]<-23+i*6)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(np.arange(-26,30,6),ave)\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"fillna(23)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['dew_temperature'] = df_train['dew_temperature'].fillna(23)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# precip_depth_1_hr\nfeature = \"precip_depth_1_hr\"\nplt.figure(figsize=(16,5))\nave = []\nnum = []\nfor i in range(9):\n    ave.append(df_train.loc[(df_train[feature]>-50+i*50) & (df_train[feature]<0+i*50)].meter_reading.mean())\n    num.append(df_train.loc[(df_train[feature]>-50+i*50) & (df_train[feature]<0+i*50)].meter_reading.count())\nplt.subplot(1,2,1)\nplt.title('mean of meter_reading')\nplt.bar(np.arange(-50,400,50),ave,width=40)\nplt.subplot(1,2,2)\nplt.title('number of precip_depth_1_hr')\nplt.bar(np.arange(-50,400,50),num,width=40)\nplt.show()\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))\nprint('number of 0:{}'.format(df_train.loc[(df_train[feature]==0)].meter_reading.count()))\nprint(\"mean of 'meter_reading when 'number is 0:{}\".format(df_train.loc[(df_train[feature]==0)].meter_reading.mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* the number of \"precip_depth_1_hr\" being 0 is too much. The mean of 'meter_reading' when 'number' is 0 is 2272 so 0 information is important I guess.\n* the distribution above is without 0. I use \"<\" not \"<=\"\n\nadd a column \"precip_depth_1_hr_ifnan\" and \"precip_depth_1_hr_ifzero\"\nfillna(300)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['precip_depth_1_hr_ifnan'] = df_train.precip_depth_1_hr.isnull().astype(\"int\")\ndf_train['precip_depth_1_hr'] = df_train['precip_depth_1_hr'].fillna(300)\n# I do not know why how to create a new column \"precip_depth_1_hr_ifzero\"\n# df_train['precip_depth_1_hr_ifzero'] = df_train.apply(lambda x:0 if x.precip_depth_1_hr==0 else 1,axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sea_level_pressure\nfeature = \"sea_level_pressure\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(8):\n    ave.append(df_train.loc[(df_train[feature]>970+i*10) & (df_train[feature]<980+i*10)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(np.arange(970,1050,10),ave)\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"fillna(980)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['sea_level_pressure'] = df_train['sea_level_pressure'].fillna(980)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# wind_direction\nfeature = \"wind_direction\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(12):\n    ave.append(df_train.loc[(df_train[feature]>=i*30) & (df_train[feature]<30+i*30)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.plot(np.arange(0,360,30),ave)\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"fillna(0)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['wind_direction'] = df_train['wind_direction'].fillna(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# wind_spped\nfeature = \"wind_speed\"\nplt.subplot(1,2,1)\ndf_train[feature].plot(kind='hist',figsize=(16,5))\nave = []\nfor i in range(7):\n    ave.append(df_train.loc[(df_train[feature]>=i*2.5) & (df_train[feature]<2.5+i*2.5)].meter_reading.mean())\nplt.subplot(1,2,2)\nplt.bar(np.arange(0,17.5,2.5),ave,width=1.5)\nplt.title('mean of meter_reading')\nprint(\"number of nan in '{}':{},pencent:{}%\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].count(),100*df_train.meter_reading.loc[df_train[feature].isnull()].count()/len(df_train)))\nprint(\"mean of 'meter_reading' when '{}' is nan:{}\".format(feature,df_train.meter_reading.loc[df_train[feature].isnull()].mean()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"fillna(15)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['wind_speed'] = df_train['wind_speed'].fillna(15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# timestamp\ndf_train.timestamp = pd.to_datetime(df_train.timestamp, format=\"%Y-%m-%d %H:%M:%S\")\ndf_train[\"hour\"] = df_train.timestamp.dt.hour\ndf_train['year'] = df_train['timestamp'].dt.year\ndf_train['month'] = df_train['timestamp'].dt.month\ndf_train['day'] = df_train['timestamp'].dt.day\ndf_train[\"weekday\"] = df_train.timestamp.dt.weekday\ndf_train['age'] = (df_train['year'] - df_train['year_built'])\ndf_train = df_train.drop('timestamp',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = reduce_mem_usage(df_train)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## lightgbm"},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lbt\n\n# fraction = 0.8\n# y_train = np.log1p(df_train['meter_reading'][0:int(fraction*len(df_train))])\n# X_train = df_train.drop('meter_reading',axis=1)[0:int(fraction*len(df_train))]\n# y_valid = np.log1p(df_train['meter_reading'][int(fraction*len(df_train)):])\n# X_valid = df_train.drop('meter_reading',axis=1)[int(fraction*len(df_train)):]\n# len(X_train),len(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import lightgbm as lgb\n\n# categorical_features = ['year_built_ifnan', 'floor_count_ifnan',\n#        'cloud_coverage_ifnan', 'primary_use_encoded',\n#        'precip_depth_1_hr_ifnan', 'hour', 'year', 'month', 'day', 'weekday']\n# train_set = lgb.Dataset(X_train, label=y_train, categorical_feature=categorical_features, free_raw_data=False)\n# valid_set = lgb.Dataset(X_valid, label=y_valid, categorical_feature=categorical_features, free_raw_data=False)\n\n\n# params = {\n#     \"objective\": \"regression\",\n#     \"boosting\": \"gbdt\",#dart,gbdt\n#     \"num_leaves\": 45,\n#     \"learning_rate\": 0.05,\n#     \"feature_fraction\": 0.9,\n#     \"reg_lambda\": 2,\n#     \"metric\": \"rmse\"\n# }\n\n# model = lgb.train(params, train_set=train_set, num_boost_round=2000, valid_sets=valid_set, verbose_eval=200, early_stopping_rounds=200)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.save_model('/kaggle/working/lightgbm_cat.txt')\n# # del X_train,y_train,X_valid,y_train,train_set,valid_set\n# # gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del _i1,_i2,_i3,_i4,_i5,_i6,_i7,_i8,_i9,_i10;\ndel _i11,_i12,_i13,_i14,_i15,_i16,_i17,_i18,_i19,_i20;\ndel _i21,_i22,_i23,_i24,_i25,_i26,_i27,_i28,_i29,_i30;\ndel _i31,_i32,_i33,_i34,_i35,_i36,_i37,_i38,_i39,_i40;\n# del X_train,y_train,X_valid,y_valid;\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## half and half"},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_train = df_train.drop('meter_reading',axis=1)\n# y_train = np.log1p(df_train.meter_reading)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_half_1 = X_train[:int(X_train.shape[0] / 2)]\n# X_half_2 = X_train[int(X_train.shape[0] / 2):]\n# y_half_1 = y_train[:int(y_train.shape[0] / 2)]\n# y_half_2 = y_train[int(y_train.shape[0] / 2):]\n# categorical_features = ['year_built_ifnan', 'floor_count_ifnan',\n#        'cloud_coverage_ifnan', 'primary_use_encoded',\n#        'precip_depth_1_hr_ifnan', 'hour', 'year', 'month', 'day', 'weekday']\n\n# d_half_1 = lgb.Dataset(X_half_1, label=y_half_1, categorical_feature=categorical_features, free_raw_data=False)\n# d_half_2 = lgb.Dataset(X_half_2, label=y_half_2, categorical_feature=categorical_features, free_raw_data=False)\n# watchlist_1 = [d_half_2, d_half_1]\n# watchlist_2 = [d_half_1, d_half_2]\n# params = {\n#     \"objective\": \"regression\",\n#     \"boosting\": \"gbdt\",#dart,gbdt\n#     \"num_leaves\": 45,\n#     \"learning_rate\": 0.02,\n#     \"feature_fraction\": 0.9,\n#     \"reg_lambda\": 2,\n#     \"metric\": \"rmse\"\n# }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print(\"Building model with first half and validating on second half:\")\n# model_half_1 = lgb.train(params, train_set=d_half_1, num_boost_round=1000, valid_sets=watchlist_1, verbose_eval=200, early_stopping_rounds=200)\n\n# print(\"Building model with second half and validating on first half:\")\n# model_half_2 = lgb.train(params, train_set=d_half_2, num_boost_round=1000, valid_sets=watchlist_2, verbose_eval=200, early_stopping_rounds=200)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_half_1.save_model('/kaggle/working/model_half_1.txt')\n# model_half_2.save_model('/kaggle/working/model_half_2.txt')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## RNN (LSTM)"},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fraction = 0.8\n# y_train = np.log1p(df_train['meter_reading'][0:int(fraction*len(df_train))])\n# X_train = df_train.drop('meter_reading',axis=1)[0:int(fraction*len(df_train))]\n# y_valid = np.log1p(df_train['meter_reading'][int(fraction*len(df_train)):])\n# X_valid = df_train.drop('meter_reading',axis=1)[int(fraction*len(df_train)):]\n# len(X_train),len(X_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# X_train = np.reshape(X_train.values,[-1,1,22])\n# X_valid = np.reshape(X_valid.values,[-1,1,22])\n# y_train = np.reshape(y_train.values,[-1,1,1])\n# y_valid = np.reshape(y_valid.values,[-1,1,1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras import Sequential\n# from keras.preprocessing.sequence import pad_sequences\n# from sklearn.model_selection import train_test_split\n# from keras.models import Sequential, Model\n# from keras.layers import LSTM, Dense, Bidirectional, Input, Dropout, BatchNormalization\n# from keras import backend as K\n# from keras.engine.topology import Layer\n# from keras import initializers, regularizers, constraints","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras import backend as K\n# def estimate(y_valid,y_pred):\n#     l = K.int_shape(y_pred)\n#     return K.pow(K.sum(K.pow(K.log(y_valid+1)-K.log(y_pred+1),2)),0.5)\n# def loss(y_valid,y_pred):\n#     l = K.int_shape(y_pred)\n#     return K.pow(K.sum(K.pow(K.log(y_valid+1)-K.log(y_pred+1),2)),0.5)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model = Sequential()\n# model.add(BatchNormalization(input_shape=(1,22)))\n# model.add(Bidirectional(LSTM(32,dropout=0.4,recurrent_dropout=0.4,activation='tanh',return_sequences=True)))\n# model.add(Bidirectional(LSTM(64,return_sequences=True)))\n# model.add(BatchNormalization(input_shape=(1,32)))\n# model.add(Bidirectional(LSTM(128, activation='tanh',return_sequences=True)))\n# model.add(BatchNormalization(input_shape=(1,64)))\n# model.add(Dense(512,activation='relu'))\n# model.add(Dropout(0.3))\n# model.add(Dense(512,activation=\"relu\"))\n# model.add(Dropout(0.3))\n# model.add(Dense(1,activation=\"softmax\"))\n# model.compile(loss='mse',optimizer='adam',metrics=['mse'])\n# print(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# history = model.fit(X_train,y_train,batch_size=300,epochs=15,\n#                    validation_data=(X_valid,y_valid))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"tips：载入整个模型结构时，若模型训练时有自定义loss或metrics，则载入时会报类似错：Unknown metric function:my_loss （此处my_loss是一个自定义函数），则加载模型时需要指定custom_objects参数：\ntips: if you cumstomize your own loss or metrics function, it might be wrong like \"Unknown metric function:my_loss\" when you load your '.h5' model file. \nsolution : \nmodel = load_model('model.h5'，{'my_loss': my_loss})\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"# model.save('/kaggle/working/LSTM.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Plot training & validation accuracy values\n# plt.plot(history.history['mse'])\n# plt.plot(history.history['loss'])\n# plt.plot(history.history['val_mse'])\n# plt.plot(history.history['val_loss'])\n# plt.title('Model accuracy')\n# plt.ylabel('Accuracy')\n# plt.xlabel('Epoch')\n# plt.legend(['estimate', 'loss','val_estimate','val_loss'], loc='upper left')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}