{"cells":[{"metadata":{},"cell_type":"markdown","source":"This Kernel is the succeeding file for the following great kernel:\n\n>- [ASHRAE: EDA File](https://www.kaggle.com/trehansalil1/ashrae-eda)"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom scipy.stats import stats\nimport lightgbm as lgb\nfrom sklearn.metrics import f1_score, mean_squared_error, confusion_matrix\n, precision_recall_fscore_support, accuracy_score, cohen_kappa_score, f1_score, classification_report\nfrom sklearn.model_selection import train_test_split\nfrom imblearn.over_sampling import SMOTE\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = pd.read_csv('../input/ashrae-energy-prediction/weather_test.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = pd.read_csv('../input/ashrae-energy-prediction/weather_train.csv',parse_dates=['timestamp'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def reduce_mem_usage(df, verbose=True):\n    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n    start_mem = df.memory_usage().sum() / 1024**2    \n    for col in df.columns:\n        col_type = df[col].dtypes\n        if col_type in numerics:\n            c_min = df[col].min()\n            c_max = df[col].max()\n            if str(col_type)[:3] == 'int':\n                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n                    df[col] = df[col].astype(np.int8)\n                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n                    df[col] = df[col].astype(np.int16)\n                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n                    df[col] = df[col].astype(np.int32)\n                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n                    df[col] = df[col].astype(np.int64)  \n            else:\n                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n                    df[col] = df[col].astype(np.float16)\n                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n                    df[col] = df[col].astype(np.float32)\n                else:\n                    df[col] = df[col].astype(np.float64)    \n    end_mem = df.memory_usage().sum() / 1024**2\n    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = reduce_mem_usage(weather_test)\nweather_train = reduce_mem_usage(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = weather_train.groupby('site_id').apply(lambda x: x.set_index('timestamp').interpolate(method='time',limit_direction='both').reset_index()).drop(columns='site_id').reset_index().drop(columns='level_1')\n\n\nweather_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = weather_test.groupby('site_id').apply(lambda x: x.set_index('timestamp').interpolate(method='time',limit_direction='both').reset_index()).drop(columns='site_id').reset_index().drop(columns='level_1')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train['cloud_coverage'] = weather_train['cloud_coverage'].round()\nweather_train['wind_direction'] = weather_train['wind_direction'].round()\nweather_train['precip_depth_1_hr'] = weather_train['precip_depth_1_hr'].round()\n\nweather_test['cloud_coverage'] = weather_test['cloud_coverage'].round()\nweather_test['wind_direction'] = weather_test['wind_direction'].round()\nweather_test['precip_depth_1_hr'] = weather_test['precip_depth_1_hr'].round()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.groupby('site_id').mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test[weather_test['sea_level_pressure'].isnull()]['site_id'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test[weather_test['wind_direction'].isnull()]['site_id'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train['aday'] = weather_train['timestamp'].dt.day\nweather_train['amonth'] = weather_train['timestamp'].dt.month\nweather_train['ayear'] = weather_train['timestamp'].dt.year\nweather_train['ahour'] = weather_train['timestamp'].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test['aday'] = weather_test['timestamp'].dt.day\nweather_test['amonth'] = weather_test['timestamp'].dt.month\nweather_test['ayear'] = weather_test['timestamp'].dt.year\nweather_test['ahour'] = weather_test['timestamp'].dt.hour","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def sea_impute(weather_test):\n    \n    \n    df_test = weather_test.loc[weather_test.site_id==5,:].drop(columns='timestamp').reset_index().drop(columns='index')\n    df_train = weather_test.loc[weather_test.site_id!=5,:].drop(columns='timestamp').reset_index().drop(columns='index')\n    X = df_train.drop(columns='sea_level_pressure')\n    y = df_train['sea_level_pressure']\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n    model = lgb.LGBMRegressor()\n    model.fit(X_train,y_train)\n    y_pred = model.predict(X_test)\n\n    print('The mean squared error for sea_level_pressure is: ',mean_squared_error(y_test, y_pred))\n\n    model.fit(X ,y)\n    y_pred = model.predict(df_test.drop(columns='sea_level_pressure'))\n    df_test['sea_level_pressure'] = pd.Series(y_pred)\n    df = pd.merge(df_train, df_test, how='outer')\n\n    del df_test, df_train, X_train, X_test, y_train, y_test, y_pred, X, y\n\n    weather_test = df\n    \n    return(weather_test)\n\ndef cloud_impute(weather_test):\n    from sklearn.metrics import classification_report, cohen_kappa_score\n    df_test = weather_test.loc[(weather_test.site_id == 7) | (weather_test.site_id == 11), :].reset_index().drop(columns='index')\n    df_train = weather_test.loc[(weather_test.site_id != 7) & (weather_test.site_id != 11), :].reset_index().drop(columns='index')\n    df_test.dropna(subset=['precip_depth_1_hr'], inplace=True)\n    df_train.dropna(subset=['precip_depth_1_hr'], inplace=True)\n    X = df_train.drop(columns='cloud_coverage').dropna()\n    y = df_train['cloud_coverage']\n    smote = SMOTE('minority')\n    X, y = smote.fit_sample(X, y)\n\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=123)\n    model = lgb.LGBMClassifier(objective='multiclass', reg_alpha= 1, reg_lambda=5, learning_rate=1.5, n_extimator=50)\n    model.fit(X_train,y_train)\n    y_pred = model.predict(X_test)\n    print('The cohen_kappa_score & f1_score for cloud_coverage is: ',cohen_kappa_score(y_test, y_pred)\n          , f1_score(y_test, y_pred, average='micro'))\n    print('Report for cloud_coverage: ')\n    print(classification_report(y_test, y_pred))\n    model.fit(X, y)\n    y_pred = model.predict(df_test.drop(columns='cloud_coverage'))\n\n    df_test['cloud_coverage'] = pd.Series(y_pred)\n    df = pd.merge(df_train,df_test, how='outer')\n    k = weather_test.loc[(weather_test.site_id == 1) | (weather_test.site_id == 12) | (weather_test.site_id == 5)\n                         , :].reset_index().drop(columns='index')\n\n    df = pd.merge(df,k, how='outer')\n\n    del df_test, df_train, X_train, X_test, y_train, y_test, y_pred, X, y, k\n\n    weather_test = df\n    \n    return(weather_test)\n\ndef precip_impute(weather_test):\n    df_test = weather_test.loc[(weather_test.site_id == 1) | (weather_test.site_id == 12) | (weather_test.site_id == 5), :].reset_index().drop(columns='index')\n    df_train = weather_test.loc[(weather_test.site_id != 1) & (weather_test.site_id != 12) & (weather_test.site_id != 5), :].reset_index().drop(columns='index')\n    X = df_train.drop(columns='precip_depth_1_hr')\n    y = df_train['precip_depth_1_hr']\n\n    X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=12)\n    model = lgb.LGBMRegressor(learning_rate=0.08, n_estimators=10000, reg_alpha=1\n                              , reg_lambda=2, lambda_l1 = 0.3)\n    model.fit(X_train,y_train)\n    y_pred = model.predict(X_test)\n    print('The mean squared error for precip_depth_1_hr is: ', mean_squared_error(y_test, y_pred))\n    y_pred = np.array([round(i) for i in y_pred])\n    print('The corrected mean squared error for precip_depth_1_hr is: ', mean_squared_error(y_test, y_pred))\n\n    model.fit(X,y)\n    y_pred = model.predict(df_test.drop(columns='precip_depth_1_hr'))\n    y_pred = np.array([round(i) for i in y_pred])\n\n    df_test['precip_depth_1_hr'] = pd.Series(y_pred)\n    df = pd.merge(df_train, df_test, how='outer')\n    #del df_test, df_train, X_train, X_test, y_train, y_test, y_pred, X, y\n\n    weather_test = df\n    \n    return(weather_test)\n\n#del df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = sea_impute(weather_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = cloud_impute(weather_test)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test = precip_impute(weather_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.groupby('site_id').mean().sort_values(by='cloud_coverage')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.isnull().sum()[weather_test.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = sea_impute(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = cloud_impute(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train = precip_impute(weather_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.isnull().sum()[weather_train.isnull().sum()!=0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_test.to_csv('weather_test_s.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"weather_train.to_csv('weather_train_s.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.4"}},"nbformat":4,"nbformat_minor":1}